From 7535905ee3141a63942c743ae978cf3020c06e67 Mon Sep 17 00:00:00 2001 From: alanhuangyoo Date: Fri, 28 Aug 2026 21:55:21 +0800 Subject: [PATCH] Read rope_theta from rope_parameters across Inference V2 transformers 5.0 folded the rotary settings into config.rope_parameters and dropped the rope_theta attribute. Eight V2 models still read the attribute: llama_v2, mistral, mixtral, phi, phi3, qwen_v2, qwen_v2_moe self._config.rope_theta -> AttributeError on 5.x exaone4 getattr(self._config, "rope_theta", 1000000.0) -> no error, but 1e6 instead of the 1e4 the config carries exaone4 is the worse of the two: a 100x rotary base is a silent numerical error, not a startup failure. exaone4_5 already reads both spellings, added with the model in #8121. Hoist the same lookup onto DSTransformerModelBase so every model that reaches it through the base gets it, and raise instead of guessing a base when neither spelling carries one. Same drift as #8341, which covers the v1 kernel-injection policy. Signed-off-by: alanhuangyoo --- .../v2/model_implementations/exaone4/model.py | 3 +- .../inference_transformer_base.py | 20 +++++ .../model_implementations/llama_v2/model.py | 2 +- .../v2/model_implementations/mistral/model.py | 2 +- .../v2/model_implementations/mixtral/model.py | 2 +- .../v2/model_implementations/phi/model.py | 2 +- .../v2/model_implementations/phi3/model.py | 2 +- .../v2/model_implementations/qwen_v2/model.py | 2 +- .../qwen_v2_moe/model.py | 2 +- .../model_implementations/test_rope_theta.py | 80 +++++++++++++++++++ 10 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 tests/unit/inference/v2/model_implementations/test_rope_theta.py diff --git a/deepspeed/inference/v2/model_implementations/exaone4/model.py b/deepspeed/inference/v2/model_implementations/exaone4/model.py index 5aeb87c2cdac..e12f5a513072 100644 --- a/deepspeed/inference/v2/model_implementations/exaone4/model.py +++ b/deepspeed/inference/v2/model_implementations/exaone4/model.py @@ -94,8 +94,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - rope_theta = getattr(self._config, "rope_theta", 1000000.0) - return RotateHalfConfig(theta_base=rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py index fae67dc8fc2a..0aabfda2e21c 100644 --- a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py +++ b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py @@ -165,6 +165,26 @@ def positional_embedding_config(self) -> Optional[RotateHalfConfig]: Derived helpers """ + @property + def rope_theta(self) -> float: + """The rotary base, read from wherever the installed transformers keeps it. + + transformers 5.0 folded the rotary settings into ``config.rope_parameters`` and + dropped the ``rope_theta`` attribute, so reading the attribute alone raises + against a stock config on 5.x. ``exaone4_5`` already reads both spellings; this + is the same lookup for every model that reaches it through this base. + """ + theta = getattr(self._config, "rope_theta", None) + if theta is not None: + return theta + + rope_parameters = getattr(self._config, "rope_parameters", None) or getattr(self._config, "rope_scaling", None) + if isinstance(rope_parameters, dict) and rope_parameters.get("rope_theta") is not None: + return rope_parameters["rope_theta"] + + raise ValueError(f"{type(self._config).__name__} carries no rope_theta, either as an " + "attribute or in rope_parameters/rope_scaling.") + @cached_property def n_heads_q_local(self) -> int: """ diff --git a/deepspeed/inference/v2/model_implementations/llama_v2/model.py b/deepspeed/inference/v2/model_implementations/llama_v2/model.py index a0c81f4d749e..ac23145e942e 100644 --- a/deepspeed/inference/v2/model_implementations/llama_v2/model.py +++ b/deepspeed/inference/v2/model_implementations/llama_v2/model.py @@ -107,7 +107,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/mistral/model.py b/deepspeed/inference/v2/model_implementations/mistral/model.py index 318d362f1a64..89289a9f171a 100644 --- a/deepspeed/inference/v2/model_implementations/mistral/model.py +++ b/deepspeed/inference/v2/model_implementations/mistral/model.py @@ -106,7 +106,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/mixtral/model.py b/deepspeed/inference/v2/model_implementations/mixtral/model.py index 878cd8e31cec..5a512fc0633a 100644 --- a/deepspeed/inference/v2/model_implementations/mixtral/model.py +++ b/deepspeed/inference/v2/model_implementations/mixtral/model.py @@ -114,7 +114,7 @@ def positional_embedding_config(self) -> Optional[RotateHalfConfig]: """ The positional embedding configuration for the model. """ - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Inherited from `DSMoETransformerModelBase` diff --git a/deepspeed/inference/v2/model_implementations/phi/model.py b/deepspeed/inference/v2/model_implementations/phi/model.py index 2d5826810cb5..41e0e8652d60 100644 --- a/deepspeed/inference/v2/model_implementations/phi/model.py +++ b/deepspeed/inference/v2/model_implementations/phi/model.py @@ -97,7 +97,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: rotary_dim = int(self._config.partial_rotary_factor * self.head_size) - return RotateHalfConfig(rotate_dim=rotary_dim, theta_base=self._config.rope_theta) + return RotateHalfConfig(rotate_dim=rotary_dim, theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/phi3/model.py b/deepspeed/inference/v2/model_implementations/phi3/model.py index 507bb4fc9af1..7d974d3a113e 100644 --- a/deepspeed/inference/v2/model_implementations/phi3/model.py +++ b/deepspeed/inference/v2/model_implementations/phi3/model.py @@ -106,7 +106,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/qwen_v2/model.py b/deepspeed/inference/v2/model_implementations/qwen_v2/model.py index d535462a954d..c89ed8a65c65 100644 --- a/deepspeed/inference/v2/model_implementations/qwen_v2/model.py +++ b/deepspeed/inference/v2/model_implementations/qwen_v2/model.py @@ -100,7 +100,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) def make_norm_layer(self) -> None: """ diff --git a/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py b/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py index c7841b24e5fc..bc441452ce70 100644 --- a/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py +++ b/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py @@ -105,7 +105,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Inherited from `DSMoETransformerModelBase` diff --git a/tests/unit/inference/v2/model_implementations/test_rope_theta.py b/tests/unit/inference/v2/model_implementations/test_rope_theta.py new file mode 100644 index 000000000000..2fda33c2b5f7 --- /dev/null +++ b/tests/unit/inference/v2/model_implementations/test_rope_theta.py @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Inference V2 must find rope_theta wherever the installed transformers keeps it. + +transformers 5.0 folded the rotary settings into ``config.rope_parameters`` and dropped +the ``rope_theta`` attribute. Models that read the attribute directly raise against a +stock config on 5.x; ``exaone4`` read it through a ``getattr`` default and silently used +that default instead of the value the config actually carries. +""" + +from types import SimpleNamespace + +import pytest + +from deepspeed.inference.v2.model_implementations.inference_transformer_base import DSTransformerModelBase + + +class _Model: + """Minimal stand-in that borrows the property under test.""" + + rope_theta = DSTransformerModelBase.rope_theta + + def __init__(self, config): + self._config = config + + +def test_reads_the_legacy_attribute(): + assert _Model(SimpleNamespace(rope_theta=500000.0)).rope_theta == 500000.0 + + +def test_reads_rope_parameters_when_the_attribute_is_gone(): + config = SimpleNamespace(rope_parameters={"rope_theta": 10000.0, "rope_type": "default"}) + + assert _Model(config).rope_theta == 10000.0 + + +def test_reads_rope_scaling_when_that_is_the_only_dict(): + config = SimpleNamespace(rope_scaling={"rope_theta": 1000000.0, "rope_type": "default"}) + + assert _Model(config).rope_theta == 1000000.0 + + +def test_prefers_the_attribute_when_both_are_present(): + config = SimpleNamespace(rope_theta=500000.0, rope_parameters={"rope_theta": 10000.0}) + + assert _Model(config).rope_theta == 500000.0 + + +def test_raises_instead_of_guessing_when_nothing_carries_it(): + # exaone4 used to fall back to 1e6 here, which is a silent 100x error against a + # config whose real base is 1e4. + with pytest.raises(ValueError, match="rope_theta"): + _ = _Model(SimpleNamespace()).rope_theta + + +@pytest.mark.parametrize( + "module_name, config_name", + [ + ("llama", "LlamaConfig"), + ("mistral", "MistralConfig"), + ("mixtral", "MixtralConfig"), + ("phi", "PhiConfig"), + ("phi3", "Phi3Config"), + ("qwen2", "Qwen2Config"), + ("qwen2_moe", "Qwen2MoeConfig"), + ("exaone4", "Exaone4Config"), + ], +) +def test_resolves_against_the_installed_transformers_configs(module_name, config_name): + """Every config backing a V2 model must yield a base through one spelling or the other.""" + importlib = pytest.importorskip("importlib") + try: + module = importlib.import_module(f"transformers.models.{module_name}.configuration_{module_name}") + config = getattr(module, config_name)() + except (ImportError, AttributeError): + pytest.skip(f"{config_name} is not available in the installed transformers") + + assert _Model(config).rope_theta > 0