style: run linting and static typing

Author: stevenhsd

src/dve/core_engine/backends/base/rules.py

@@ -45,10 +45,7 @@
     TableUnion,
 )
 from dve.core_engine.backends.types import Entities, EntityType, StageSuccessful
-from dve.core_engine.configuration.v1.hierarchy import (
-    EntityHierarchy,
-    HierarchyNode,
-)
+from dve.core_engine.configuration.v1.hierarchy import EntityHierarchy, HierarchyNode
 from dve.core_engine.constants import ORPHANED_RECORD_ENTITY_NAME
 from dve.core_engine.exceptions import CriticalProcessingError
 from dve.core_engine.loggers import get_logger
@@ -418,7 +415,9 @@ def process_node(
                 )
 
                 if no_orphs > 0:
-                    self.logger.info(f"Removing records with missing parent from {current_entity_name}")
+                    self.logger.info(
+                        f"Removing records with missing parent from {current_entity_name}"
+                    )
                     location = list(node.join_fields.values())[0]
                     with BackgroundMessageWriter(
                         working_directory=working_directory,
@@ -435,23 +434,26 @@ def process_node(
                                     code=node.missing_parent_id_error_code,
                                     message=node.missing_parent_id_error_message,
                                     location=location,
+                                ),
+                            ),
+                        )
+                        # moved to batch the write - risky if large number of
+                        msg_writer.write_queue.put(
+                            [
+                                FeedbackMessage(
+                                    entity=current_entity_name,
+                                    record=record,  # type: ignore
+                                    error_location=location,
+                                    error_message=node.missing_parent_id_error_message,
+                                    failure_type="record",
+                                    error_type="record",
+                                    error_code=node.missing_parent_id_error_code,
+                                    reporting_field=location,
+                                    category="Parent Missing",
                                 )
-                            )
+                                for record in _orph_records
+                            ]
                         )
-                    # moved to batch the write - risky if large number of
-                        msg_writer.write_queue.put([
-                            FeedbackMessage(
-                                entity=current_entity_name,
-                                record=record,  # type: ignore
-                                error_location=location,
-                                error_message=node.missing_parent_id_error_message,
-                                failure_type="record",
-                                error_type="record",
-                                error_code=node.missing_parent_id_error_code,
-                                reporting_field=location,
-                                category="Parent Missing",
-                            )
-                        for record in _orph_records ])
 
             if node.children:
                 for child_node in node.children:

src/dve/core_engine/backends/metadata/rules.py

@@ -553,6 +553,7 @@ class OrphanIdentification(AbstractConditionalJoin):
 
     """
 
+
 Step = Union[AbstractStep, Literal["sync"]]
 """A step within a rule. This is either a rule config or the literal string 'sync'."""
 

src/dve/core_engine/configuration/v1/__init__.py

@@ -106,29 +106,31 @@ class _LinkageConfig(BaseModel):
     """The error code to emit if the entity has no valid records and is mandatory in the parent entity"""  # pylint: disable=C0301
     no_valid_records_error_message: Optional[ErrorMessage] = (
         "parent record removed as no valid child records"
-        )
+    )
     """The error message to emit if the entity has no valid records and is mandatory in the parent entity"""  # pylint: disable=C0301
     missing_parent_id_error_code: Optional[ErrorCode] = "MissingParentRecord"
     """The error code to emit if the entity contains records that are orphaned by parent record rejections"""  # pylint: disable=C0301
     missing_parent_id_error_message: Optional[ErrorMessage] = (
         "Records removed due to no valid parent record"
     )
     """The error code to emit if the entity contains records that are orphaned by parent record rejections"""  # pylint: disable=C0301
-       
+
     @model_validator(mode="after")
     def _check_root_no_parent_or_join_keys(self):
         if self.is_root_entity:
             if self.parent_entity or self.join_fields:
-                raise ValueError("If entity is root, neither parent_entity nor join keys should be specified")
+                raise ValueError(
+                    "If entity is root, neither parent_entity nor join keys should be specified"
+                )
         return self
-    
+
     @model_validator(mode="after")
     def _check_root_mandatory(self):
         if self.is_root_entity:
             if not self.mandatory:
                 raise ValueError("If entity is root, it must be labelled mandatory")
         return self
-    
+
     @model_validator(mode="after")
     def _check_parent_entity_with_join_keys(self):
         if self.parent_entity or self.join_fields:

src/dve/core_engine/configuration/v1/hierarchy.py

@@ -3,7 +3,7 @@
 import json
 from typing import Any, Iterable, Optional, Union
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field
 
 from dve.core_engine.configuration.v1 import V1EngineConfig, _LinkageConfig
 from dve.core_engine.type_hints import EntityName, ErrorCode, ErrorMessage
@@ -16,18 +16,16 @@ class HierarchyNode(BaseModel):
     """Stores entity hierarchy information"""
 
     entity_name: str
-    children: Optional[list["HierarchyNode"]] = Field(default_factory=list)
-    mandatory: Optional[bool] = False
-    join_fields: Optional[dict[str, str]] = Field(default_factory=dict)
-    no_valid_records_error_code: Optional[ErrorCode] = "NoValidRecords"
-    no_valid_records_error_message: Optional[ErrorMessage] = (
-        "parent record removed as no valid child records"
-    )
+    children: list["HierarchyNode"] = Field(default_factory=list)
+    mandatory: bool = False
+    join_fields: dict[str, str] = Field(default_factory=dict)
+    no_valid_records_error_code: ErrorCode = "NoValidRecords"
+    no_valid_records_error_message: ErrorMessage = "parent record removed as no valid child records"
     missing_parent_id_error_code: Optional[ErrorCode] = "MissingParentRecord"
     missing_parent_id_error_message: Optional[ErrorMessage] = (
         "Records removed due to no valid parent record"
     )
-    
+
     def get_descendents(self) -> list[str]:
         """Recursively list all descendents of the node"""
         descendents = []
@@ -79,24 +77,34 @@ def determine_trees(
         all_datasets: Iterable[str], entity_relationships: dict[str, _LinkageConfig]
     ) -> dict[EntityName, HierarchyNode]:
         """Determine the entity hierarchy trees and store as HierarchyNodes"""
-        root_entities: dict[str, _LinkageConfig] = dict(filter(lambda x: x[1].is_root_entity,
-                                                        entity_relationships.items()))
+        root_entities: dict[str, _LinkageConfig] = dict(
+            filter(lambda x: x[1].is_root_entity, entity_relationships.items())
+        )
         top_level_parents: dict[EntityName, HierarchyNode] = {
-            entity_name: HierarchyNode(entity_name=entity_name,
-                                       **config.model_dump(exclude={"parent_entity",
-                                                                    "missing_parent_id_error_code",
-                                                                    "missing_parent_id_error_message"}),
-                                       missing_parent_id_error_code=None,
-                                       missing_parent_id_error_message=None)
+            entity_name: HierarchyNode(
+                entity_name=entity_name,
+                **config.model_dump(
+                    exclude={
+                        "parent_entity",
+                        "missing_parent_id_error_code",
+                        "missing_parent_id_error_message",
+                    }
+                ),
+                missing_parent_id_error_code=None,
+                missing_parent_id_error_message=None,
+            )
             for entity_name, config in root_entities.items()
         }
-        
-        if default_roots := [ entity_name for entity_name in all_datasets 
-                             if not entity_name in entity_relationships]:
+
+        if default_roots := [
+            entity_name for entity_name in all_datasets if not entity_name in entity_relationships
+        ]:
             for entity_name in default_roots:
-                top_level_parents[entity_name] = HierarchyNode(entity_name=entity_name,
-                                                               missing_parent_id_error_code=None,
-                                                               missing_parent_id_error_message=None)
+                top_level_parents[entity_name] = HierarchyNode(
+                    entity_name=entity_name,
+                    missing_parent_id_error_code=None,
+                    missing_parent_id_error_message=None,
+                )
 
         for name, linkage_detail in entity_relationships.items():
             for main_entity, parent_node in top_level_parents.items():