perf(pt_expt): use scalable graph builders during training

deepmd/pt/optimizer/hybrid_muon.py

@@ -6,8 +6,8 @@
 Routing is controlled by parameter dimensionality, parameter names, and
 ``muon_mode``:
 
-- Parameters whose final effective name segment contains ``bias``
-  (case-insensitive), or starts with ``adam_`` (case-insensitive): Adam.
+- Parameters whose final effective name segment is ``b``, contains ``bias``,
+  or starts with ``adam_`` (case-insensitive): Adam.
 - Parameters whose final effective name segment starts with ``adamw_``
   (case-insensitive): Adam with decoupled weight decay (AdamW-style).
   The final effective segment means the last non-numeric segment in the full
@@ -715,7 +715,7 @@ def get_adam_route(
     effective name segment after stripping trailing numeric ParameterList
     indices):
 
-    1. Contains ``"bias"`` -> ``"adam"`` (no weight decay).
+    1. Is ``"b"`` or contains ``"bias"`` -> ``"adam"`` (no weight decay).
     2. Starts with ``"adam_"`` -> ``"adam"`` (no weight decay).
        Typical: norm scales, radial frequencies.
     3. Starts with ``"adamw_"`` -> ``"adamw"`` (decoupled weight decay).
@@ -730,7 +730,7 @@ def get_adam_route(
     while leaf_name_idx > 0 and name_segments[leaf_name_idx].isdigit():
         leaf_name_idx -= 1
     leaf_name = name_segments[leaf_name_idx]
-    if "bias" in leaf_name:
+    if leaf_name == "b" or "bias" in leaf_name:
         return "adam"
     if leaf_name.startswith("adam_"):
         return "adam"
@@ -808,9 +808,9 @@ class HybridMuonOptimizer(Optimizer):
 
     This optimizer applies different update rules based on parameter dimensionality,
     parameter names, and ``muon_mode``:
-    - Parameters with final effective name segment containing ``bias``
-      (case-insensitive), or starting with ``adam_`` (case-insensitive):
-      standard Adam update.
+    - Parameters with final effective name segment equal to ``b``, containing
+      ``bias``, or starting with ``adam_`` (case-insensitive): standard Adam
+      update.
     - Parameters with final effective name segment starting with ``adamw_``
       (case-insensitive): Adam with decoupled weight decay (AdamW-style).
     - 1D parameters: standard Adam update.
@@ -826,8 +826,8 @@ class HybridMuonOptimizer(Optimizer):
         ``(m, n)`` slice.
 
     Naming convention for explicit Adam routing:
-    - Parameters representing bias terms should include ``bias`` in their
-      final effective name segment (case-insensitive).
+    - Parameters representing bias terms should use ``b`` or include ``bias``
+      in their final effective name segment (case-insensitive).
     - Parameters that are not semantic bias but should still use Adam should
       use an ``adam_`` prefix in their final effective name segment
       (case-insensitive).
@@ -887,10 +887,10 @@ class HybridMuonOptimizer(Optimizer):
         - ``"slice"``: >=3D parameters use per-slice Muon routing on last two dims.
     named_parameters : iterable[tuple[str, torch.Tensor]] | None
         Optional named parameter iterable used for name-based routing.
-        Parameters with final effective name segment containing ``bias``
-        (case-insensitive), or starting with ``adam_`` (case-insensitive),
-        are forced to Adam (no weight decay). Parameters starting with
-        ``adamw_`` are forced to AdamW-style decoupled decay path.
+        Parameters with final effective name segment equal to ``b``, containing
+        ``bias``, or starting with ``adam_`` (case-insensitive) are forced to
+        Adam (no weight decay). Parameters starting with ``adamw_`` are forced
+        to AdamW-style decoupled decay path.
     enable_gram : bool
         Enable the compiled Gram Newton-Schulz path for rectangular Muon
         matrices. Square matrices continue to use the current standard
@@ -1502,7 +1502,7 @@ def _build_param_routing(self) -> None:
         Classify parameters into Muon, Adam, and AdamW routes (static routing).
 
         Routing logic:
-        - name-based ``adam_`` prefix or contains ``bias`` → Adam (no decay)
+        - name-based ``b``, ``bias``, or ``adam_`` route → Adam (no decay)
         - name-based ``adamw_`` prefix → AdamW (decoupled weight decay)
         - effective shape rank <2 → Adam (no decay)
         - non-matrix effective shape for current muon_mode → AdamW (decoupled)

deepmd/pt/utils/compile_compat.py

@@ -84,11 +84,11 @@ def _torch_release() -> tuple[int, int]:
 def apply_global_compile_patches() -> None:
     """Apply every process-global PyTorch adjustment the compile path needs.
 
-    The adjustments are mutually independent and individually idempotent. The
-    function is intended to run exactly once, when the model module is
-    imported, so that the global state is established before the first
-    compilation. The symbolic-divisibility repair is applied only on the
-    releases where the regression exists.
+    The adjustments are mutually independent and individually idempotent.
+    Invoke this function before the first Dynamo or Inductor compilation in
+    each entry path; repeated calls from independent compile paths are safe.
+    The symbolic-divisibility repair is applied only on releases where the
+    regression exists.
     """
     # Silence Inductor / Triton autotune console dumps.  ``torch.compile``
     # reads these environment variables once, when its backend is first

deepmd/pt_expt/model/make_model.py

@@ -26,6 +26,9 @@
 from deepmd.pt_expt.common import (
     torch_module,
 )
+from deepmd.pt_expt.utils.graph_builder import (
+    build_neighbor_graph_for_method,
+)
 from deepmd.pt_expt.utils.graph_csr import (
     validate_graph_csr_for_export,
 )
@@ -249,56 +252,6 @@ def __call__(self, coord_flat: torch.Tensor) -> torch.Tensor:
         return energy_redu
 
 
-def _build_graph_for_method(
-    method: str,
-    coord: torch.Tensor,
-    atype: torch.Tensor,
-    box: torch.Tensor | None,
-    rcut: float,
-    pair_excl: Any,
-    with_csr: bool = False,
-) -> Any:
-    """Build a carry-all ``NeighborGraph`` for the named pt_expt builder.
-
-    Single owning site for the graph-builder dispatch shared by
-    :meth:`_call_common_graph` and the graph Hessian wrapper
-    (:class:`_WrapperForwardEnergyGraph`), so both build the graph identically.
-    """
-    from deepmd.dpmodel.utils.neighbor_graph import (
-        build_neighbor_graph,
-        build_neighbor_graph_ase,
-    )
-
-    if method == "dense":
-        return build_neighbor_graph(
-            coord, atype, box, rcut, with_csr=with_csr, pair_excl=pair_excl
-        )
-    if method == "ase":
-        return build_neighbor_graph_ase(
-            coord, atype, box, rcut, with_csr=with_csr, pair_excl=pair_excl
-        )
-    if method == "vesin":
-        from deepmd.pt_expt.utils.vesin_graph_builder import (
-            build_neighbor_graph_vesin,
-        )
-
-        return build_neighbor_graph_vesin(
-            coord, atype, box, rcut, with_csr=with_csr, pair_excl=pair_excl
-        )
-    if method == "nv":
-        from deepmd.pt_expt.utils.nv_graph_builder import (
-            build_neighbor_graph_nv,
-        )
-
-        return build_neighbor_graph_nv(
-            coord, atype, box, rcut, with_csr=with_csr, pair_excl=pair_excl
-        )
-    raise ValueError(
-        f"unknown neighbor_graph_method {method!r}; use 'dense', 'ase', "
-        "'vesin', or 'nv'"
-    )
-
-
 class _WrapperForwardEnergyGraph:
     """Graph twin of :class:`_WrapperForwardEnergy` for the Hessian.
 
@@ -339,7 +292,7 @@ def __init__(
 
     def __call__(self, coord_flat: torch.Tensor) -> torch.Tensor:
         cc = coord_flat.reshape(1, self.nloc, 3)
-        ng = _build_graph_for_method(
+        ng = build_neighbor_graph_for_method(
             self.method, cc, self.atype, self.box, self.rcut, self.pair_excl
         )
         atomic_ret = self.model.atomic_model.forward_common_atomic_graph(
@@ -719,7 +672,7 @@ def _resolve_graph_method(
             if "energy" not in self.atomic_output_def().keys():
                 return None
             if self.mixed_types() and self.atomic_model.uses_graph_lower():
-                return "dense"
+                return getattr(self, "neighbor_graph_method", "dense")
             return None
 
         def _call_common_graph(
@@ -795,7 +748,7 @@ def _call_common_graph(
                 and bool(getattr(_desc, "geo_compress", False))
             )
             pair_excl = getattr(self.atomic_model, "pair_excl", None)
-            ng = _build_graph_for_method(
+            ng = build_neighbor_graph_for_method(
                 method, cc, atype, bb, rcut, pair_excl, with_csr=with_csr
             )
             nf, nloc = atype.shape[:2]

deepmd/pt_expt/train/training.py

@@ -50,13 +50,21 @@
 from deepmd.dpmodel.utils.training_utils import (
     compute_total_numb_batch,
 )
+from deepmd.pt.optimizer import (
+    HybridMuonOptimizer,
+)
 from deepmd.pt.train.utils import (
     resolve_best_checkpoint_dir,
 )
 from deepmd.pt.train.validation import (
     FullValidator,
     resolve_full_validation_start_step,
 )
+from deepmd.pt.utils.compile_compat import (
+    apply_global_compile_patches,
+    build_inductor_compile_options,
+    check_compile_torch_version,
+)
 from deepmd.pt.utils.compile_compat import next_safe_prime as _next_safe_prime
 from deepmd.pt.utils.compile_compat import rebuild_graph_module as _rebuild_graph_module
 from deepmd.pt.utils.compile_compat import (
@@ -583,19 +591,14 @@ def _finalize_compiled_lower(
     if not was_training:
         model.eval()
 
-    # Inductor defaults tuned for second-order-gradient training graphs.
-    # User-supplied compile_opts override these on a per-key basis.
-    inductor_options: dict[str, Any] = {
-        "max_autotune": False,
-        "shape_padding": True,
-        "epilogue_fusion": False,
-        "triton.cudagraphs": False,
-        "max_fusion_size": 8,
-        # NOTE: On GPU with PyTorch <=2.11, consider adding
-        # "triton.mix_order_reduction": False to work around
-        # pytorch/pytorch#174379, #178080, #179494 under
-        # data-dependent symbolic shapes.
-    }
+    # This is the common boundary immediately before every pt_expt
+    # ``torch.compile`` call. Applying the idempotent process-global patches
+    # here leaves eager-only imports untouched while still preceding all
+    # Dynamo and Inductor configuration reads.
+    apply_global_compile_patches()
+
+    # Keep pt_expt training on the same compiler contract as the PT SeZM path.
+    inductor_options = build_inductor_compile_options(inference=False)
     if extra_options:
         inductor_options.update(extra_options)
     if compile_opts:
@@ -1157,8 +1160,8 @@ def _forward_graph(
         so no extended->local scatter is needed; only the flat ``(N, *)`` node
         keys are unravelled to ``(nf, nloc, *)`` at the I/O boundary.
         """
-        from deepmd.dpmodel.utils.neighbor_graph import (
-            build_neighbor_graph,
+        from deepmd.pt_expt.utils.graph_builder import (
+            build_neighbor_graph_for_method,
         )
 
         _model = self.original_model
@@ -1209,7 +1212,14 @@ def _forward_graph(
         # into edge_mask here so the compiled lower consumes a pre-excluded graph
         # (the lower no longer re-applies it), matching the eager path exactly.
         pair_excl = getattr(_model.atomic_model, "pair_excl", None)
-        ng = build_neighbor_graph(coord_3d, atype, box_flat, rcut, pair_excl=pair_excl)
+        ng = build_neighbor_graph_for_method(
+            getattr(_model, "neighbor_graph_method", "dense"),
+            coord_3d,
+            atype,
+            box_flat,
+            rcut,
+            pair_excl,
+        )
         atype_flat = atype.reshape(nframes * nloc)
 
         # Lazy compile of the GRAPH lower (cached per structure key).
@@ -1363,6 +1373,7 @@ def __init__(
 
         model_params = config["model"]
         training_params = config["training"]
+        optimizer_params = config.get("optimizer", {})
         validating_params = config.get("validating", {}) or {}
 
         # Task normalization --------------------------------------------------
@@ -1614,24 +1625,51 @@ def initialize_statistics(
                 )
 
         # Optimiser -----------------------------------------------------------
-        opt_type = training_params.get("opt_type", "Adam")
+        opt_type = optimizer_params.get("type", "Adam")
+        if opt_type not in {"Adam", "AdamW", "HybridMuon"}:
+            raise ValueError(f"Unsupported optimizer type: {opt_type}")
+
         # LambdaLR multiplies each param group's initial learning rate by the
         # lambda value.  Warmup schedules legitimately return zero at step 0,
         # so use the nonzero schedule base as the denominator and let the
         # lambda initialize the optimizer to the requested warmup value.
         initial_lr = float(self.lr_schedule.start_lr)
+        adam_betas = (
+            float(optimizer_params["adam_beta1"]),
+            float(optimizer_params["adam_beta2"]),
+        )
+        weight_decay = float(optimizer_params["weight_decay"])
 
         if opt_type == "Adam":
-            self.optimizer = torch.optim.Adam(self.wrapper.parameters(), lr=initial_lr)
+            self.optimizer = torch.optim.Adam(
+                self.wrapper.parameters(),
+                lr=initial_lr,
+                betas=adam_betas,
+                weight_decay=weight_decay,
+            )
         elif opt_type == "AdamW":
-            weight_decay = training_params.get("weight_decay", 0.001)
             self.optimizer = torch.optim.AdamW(
                 self.wrapper.parameters(),
                 lr=initial_lr,
+                betas=adam_betas,
                 weight_decay=weight_decay,
             )
-        else:
-            raise ValueError(f"Unsupported optimizer type: {opt_type}")
+        else:  # HybridMuon
+            runtime_named_parameters = tuple(self.wrapper.named_parameters())
+            self.optimizer = HybridMuonOptimizer(
+                self.wrapper.parameters(),
+                lr=initial_lr,
+                momentum=float(optimizer_params["momentum"]),
+                weight_decay=weight_decay,
+                adam_betas=adam_betas,
+                lr_adjust=float(optimizer_params["lr_adjust"]),
+                lr_adjust_coeff=float(optimizer_params["lr_adjust_coeff"]),
+                muon_mode=str(optimizer_params["muon_mode"]),
+                named_parameters=runtime_named_parameters,
+                enable_gram=bool(optimizer_params["enable_gram"]),
+                flash_muon=bool(optimizer_params["flash_muon"]),
+                magma_muon=bool(optimizer_params["magma_muon"]),
+            )
 
         for param_group in self.optimizer.param_groups:
             param_group["initial_lr"] = initial_lr
@@ -1840,9 +1878,14 @@ def update_finetune_bias(
                     last_epoch=self.start_step - 1,
                 )
 
+        self._configure_neighbor_graph_method(
+            training_params.get("neighbor_graph_method", "auto")
+        )
+
         # torch.compile -------------------------------------------------------
         self.enable_compile = training_params.get("enable_compile", False)
         if self.enable_compile:
+            check_compile_torch_version()
             compile_opts = training_params.get("compile_options", {})
             log.info("Compiling model with torch.compile (%s)", compile_opts)
             self._compile_model(compile_opts)
@@ -1938,6 +1981,29 @@ def _raise_if_full_validation_unsupported(
     # torch.compile helpers
     # ------------------------------------------------------------------
 
+    def _configure_neighbor_graph_method(self, requested: str) -> None:
+        """Resolve and install the training graph builder on eligible models."""
+        graph_models = [
+            self.models[model_key]
+            for model_key in self.model_keys
+            if model_uses_graph_lower(self.models[model_key])
+        ]
+        if not graph_models:
+            if requested != "auto":
+                raise ValueError(
+                    "training.neighbor_graph_method applies only to "
+                    "graph-eligible energy models"
+                )
+            return
+
+        from deepmd.pt_expt.utils.graph_builder import (
+            resolve_neighbor_graph_method,
+        )
+
+        resolved = resolve_neighbor_graph_method(requested, DEVICE)
+        for model in graph_models:
+            model.neighbor_graph_method = resolved
+
     def _compile_model(self, compile_opts: dict[str, Any]) -> None:
         """Replace ``self.model`` with a compiled version.
 
@@ -1953,14 +2019,6 @@ def _compile_model(self, compile_opts: dict[str, Any]) -> None:
         needed.  The coord extension + nlist build (data-dependent
         control flow) are kept outside the compiled region.
         """
-        # Disable DDPOptimizer: our compile region wraps only the inner
-        # compute function, not the whole DDP model.  DDPOptimizer assumes
-        # it owns the full model graph and splits at bucket boundaries,
-        # producing subgraphs whose outputs include symbolic integers.
-        # AOT Autograd then crashes with ``'int' object has no attribute
-        # 'meta'`` (pytorch/pytorch#134182).
-        torch._dynamo.config.optimize_ddp = False
-
         # Under DDP, self.wrapper is a DistributedDataParallel wrapper;
         # access the underlying ModelWrapper via .module.
         wrapper_mod = (