[core] deprecate torch_dtype and prefer dtype following transformers.

src/diffusers/loaders/ip_adapter.py

@@ -29,6 +29,7 @@
     is_accelerate_available,
     is_torch_version,
     is_transformers_available,
+    is_transformers_version,
     logging,
 )
 from .unet_loader_utils import _maybe_expand_lora_scales
@@ -204,13 +205,19 @@ def load_ip_adapter(
                         else:
                             image_encoder_subfolder = Path(image_encoder_folder).as_posix()
 
+                        # transformers renamed `torch_dtype` to `dtype` in 4.56.0.
+                        dtype_kwarg = (
+                            {"dtype": self.dtype}
+                            if is_transformers_version(">=", "4.56.0")
+                            else {"torch_dtype": self.dtype}
+                        )
                         image_encoder = CLIPVisionModelWithProjection.from_pretrained(
                             pretrained_model_name_or_path_or_dict,
                             subfolder=image_encoder_subfolder,
                             low_cpu_mem_usage=low_cpu_mem_usage,
                             cache_dir=cache_dir,
                             local_files_only=local_files_only,
-                            torch_dtype=self.dtype,
+                            **dtype_kwarg,
                         ).to(self.device)
                         self.register_modules(image_encoder=image_encoder)
                     else:
@@ -1043,11 +1050,17 @@ def load_ip_adapter(
                         "cache_dir": cache_dir,
                         "local_files_only": local_files_only,
                     }
+                    # transformers renamed `torch_dtype` to `dtype` in 4.56.0.
+                    dtype_kwarg = (
+                        {"dtype": self.dtype}
+                        if is_transformers_version(">=", "4.56.0")
+                        else {"torch_dtype": self.dtype}
+                    )
 
                     self.register_modules(
                         feature_extractor=SiglipImageProcessor.from_pretrained(image_encoder_subfolder, **kwargs),
                         image_encoder=SiglipVisionModel.from_pretrained(
-                            image_encoder_subfolder, torch_dtype=self.dtype, **kwargs
+                            image_encoder_subfolder, **dtype_kwarg, **kwargs
                         ).to(self.device),
                     )
                 else:

src/diffusers/modular_pipelines/modular_pipeline.py

@@ -2347,8 +2347,8 @@ def load_components(self, names: list[str] | str | None = None, **kwargs):
                    default_creation_method == "from_pretrained". If provided as a list or string, will load only the
                    specified components.
             **kwargs: additional kwargs to be passed to `from_pretrained()`.Can be:
-             - a single value to be applied to all components to be loaded, e.g. torch_dtype=torch.bfloat16
-             - a dict, e.g. torch_dtype={"unet": torch.bfloat16, "default": torch.float32}
+             - a single value to be applied to all components to be loaded, e.g. dtype=torch.bfloat16
+             - a dict, e.g. dtype={"unet": torch.bfloat16, "default": torch.float32}
              - if potentially override ComponentSpec if passed a different loading field in kwargs, e.g.
                `pretrained_model_name_or_path`, `variant`, `revision`, etc.
              - if potentially override ComponentSpec if passed a different loading field in kwargs, e.g.
@@ -2638,7 +2638,7 @@ def module_is_offloaded(module):
                     " is not recommended to move them to `cpu` as running them will fail. Please make"
                     " sure to use an accelerator to run the pipeline in inference, due to the lack of"
                     " support for`float16` operations on this device in PyTorch. Please, remove the"
-                    " `torch_dtype=torch.float16` argument, or use another device for inference."
+                    " `dtype=torch.float16` argument, or use another device for inference."
                 )
         return self
 

src/diffusers/modular_pipelines/modular_pipeline_utils.py

@@ -26,7 +26,7 @@
 
 from ..configuration_utils import ConfigMixin, FrozenDict
 from ..loaders.single_file_utils import _is_single_file_path_or_url
-from ..utils import DIFFUSERS_LOAD_ID_FIELDS, is_torch_available, logging
+from ..utils import _TORCH_DTYPE_DEPRECATION_MESSAGE, DIFFUSERS_LOAD_ID_FIELDS, deprecate, is_torch_available, logging
 from ..utils.import_utils import _is_package_available
 
 
@@ -293,6 +293,15 @@ def create(self, config: FrozenDict | dict[str, Any] | None = None, **kwargs) ->
     # YiYi TODO: add guard for type of model, if it is supported by from_pretrained
     def load(self, **kwargs) -> Any:
         """Load component using from_pretrained."""
+        torch_dtype = kwargs.pop("torch_dtype", None)
+        if torch_dtype is not None:
+            if kwargs.get("dtype") is not None:
+                raise ValueError(
+                    "You have passed both `dtype` and `torch_dtype`. Please only pass `dtype`, `torch_dtype` is deprecated."
+                )
+            deprecate("torch_dtype", "1.0.0", _TORCH_DTYPE_DEPRECATION_MESSAGE)
+            kwargs["dtype"] = torch_dtype
+
         # select loading fields from kwargs passed from user: e.g. pretrained_model_name_or_path, subfolder, variant, revision, note the list could change
         passed_loading_kwargs = {key: kwargs.pop(key) for key in self.loading_fields() if key in kwargs}
         # merge loading field value in the spec with user passed values to create load_kwargs
@@ -311,12 +320,11 @@ def load(self, **kwargs) -> Any:
 
         from diffusers import AutoModel
 
-        # `dtype`/`torch_dtype` is not an accepted parameter for tokenizers and processors.
+        # `dtype` is not an accepted parameter for tokenizers and processors.
         # As a result, it gets stored in `init_kwargs`, which are written to the config
         # during save. This causes JSON serialization to fail when saving the component.
         if self.type_hint is not None and not issubclass(self.type_hint, (torch.nn.Module, AutoModel)):
             kwargs.pop("dtype", None)
-            kwargs.pop("torch_dtype", None)
 
         if self.type_hint is None:
             try:

src/diffusers/pipelines/pipeline_loading_utils.py

@@ -563,7 +563,7 @@ def _load_empty_model(
     pipelines: Any,
     is_pipeline_module: bool,
     name: str,
-    torch_dtype: str | torch.dtype,
+    dtype: str | torch.dtype,
     cached_folder: str | os.PathLike,
     **kwargs,
 ):
@@ -636,7 +636,7 @@ def _load_empty_model(
             model = class_obj(config)
 
     if model is not None:
-        model = model.to(dtype=torch_dtype)
+        model = model.to(dtype=dtype)
     return model
 
 
@@ -677,7 +677,7 @@ def _get_final_device_map(device_map, pipeline_class, passed_class_obj, init_dic
     # To avoid circular import problem.
     from diffusers import pipelines
 
-    torch_dtype = kwargs.get("torch_dtype", torch.float32)
+    dtype = kwargs.get("dtype", torch.float32)
 
     # Load each module in the pipeline on a meta device so that we can derive the device map.
     init_empty_modules = {}
@@ -708,9 +708,7 @@ def _get_final_device_map(device_map, pipeline_class, passed_class_obj, init_dic
 
         else:
             sub_model_dtype = (
-                torch_dtype.get(name, torch_dtype.get("default", torch.float32))
-                if isinstance(torch_dtype, dict)
-                else torch_dtype
+                dtype.get(name, dtype.get("default", torch.float32)) if isinstance(dtype, dict) else dtype
             )
             loaded_sub_model = _load_empty_model(
                 library_name=library_name,
@@ -720,7 +718,7 @@ def _get_final_device_map(device_map, pipeline_class, passed_class_obj, init_dic
                 is_pipeline_module=is_pipeline_module,
                 pipeline_class=pipeline_class,
                 name=name,
-                torch_dtype=sub_model_dtype,
+                dtype=sub_model_dtype,
                 cached_folder=kwargs.get("cached_folder", None),
                 force_download=kwargs.get("force_download", None),
                 proxies=kwargs.get("proxies", None),
@@ -738,9 +736,7 @@ def _get_final_device_map(device_map, pipeline_class, passed_class_obj, init_dic
     module_sizes = {
         module_name: compute_module_sizes(
             module,
-            dtype=torch_dtype.get(module_name, torch_dtype.get("default", torch.float32))
-            if isinstance(torch_dtype, dict)
-            else torch_dtype,
+            dtype=dtype.get(module_name, dtype.get("default", torch.float32)) if isinstance(dtype, dict) else dtype,
         )[""]
         for module_name, module in init_empty_modules.items()
         if isinstance(module, torch.nn.Module)
@@ -776,7 +772,7 @@ def load_sub_model(
     pipelines: Any,
     is_pipeline_module: bool,
     pipeline_class: Any,
-    torch_dtype: torch.dtype,
+    dtype: torch.dtype,
     provider: Any,
     sess_options: Any,
     device_map: dict[str, torch.device] | str | None,
@@ -856,9 +852,9 @@ def load_sub_model(
     # For transformers models >= 4.56.0, use 'dtype' instead of 'torch_dtype' to avoid deprecation warnings
     if issubclass(class_obj, torch.nn.Module):
         if is_transformers_model and transformers_version >= version.parse("4.56.0"):
-            loading_kwargs["dtype"] = torch_dtype
+            loading_kwargs["dtype"] = dtype
         else:
-            loading_kwargs["torch_dtype"] = torch_dtype
+            loading_kwargs["torch_dtype"] = dtype
     if issubclass(class_obj, diffusers_module.OnnxRuntimeModel):
         loading_kwargs["provider"] = provider
         loading_kwargs["sess_options"] = sess_options

src/diffusers/pipelines/pipeline_utils.py

@@ -59,6 +59,7 @@
 from ..quantizers.bitsandbytes.utils import _check_bnb_status
 from ..schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
 from ..utils import (
+    _TORCH_DTYPE_DEPRECATION_MESSAGE,
     CONFIG_NAME,
     DEPRECATED_REVISION_ARGS,
     BaseOutput,
@@ -592,7 +593,7 @@ def module_is_offloaded(module):
                     " is not recommended to move them to `cpu` as running them will fail. Please make"
                     " sure to use an accelerator to run the pipeline in inference, due to the lack of"
                     " support for`float16` operations on this device in PyTorch. Please, remove the"
-                    " `torch_dtype=torch.float16` argument, or use another device for inference."
+                    " `dtype=torch.float16` argument, or use another device for inference."
                 )
         return self
 
@@ -784,7 +785,13 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike, **kwa
         from_flax = kwargs.pop("from_flax", False)
         torch_dtype = kwargs.pop("torch_dtype", None)
         dtype = kwargs.pop("dtype", None)
-        torch_dtype = dtype if dtype is not None else torch_dtype
+        if torch_dtype is not None:
+            if dtype is not None:
+                raise ValueError(
+                    "You have passed both `dtype` and `torch_dtype`. Please only pass `dtype`, `torch_dtype` is deprecated."
+                )
+            deprecate("torch_dtype", "1.0.0", _TORCH_DTYPE_DEPRECATION_MESSAGE)
+            dtype = torch_dtype
         custom_pipeline = kwargs.pop("custom_pipeline", None)
         custom_revision = kwargs.pop("custom_revision", None)
         provider = kwargs.pop("provider", None)
@@ -805,11 +812,9 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike, **kwa
         disable_mmap = kwargs.pop("disable_mmap", False)
         trust_remote_code = kwargs.pop("trust_remote_code", False)
 
-        if torch_dtype is not None and not isinstance(torch_dtype, dict) and not isinstance(torch_dtype, torch.dtype):
-            torch_dtype = torch.float32
-            logger.warning(
-                f"Passed `torch_dtype` {torch_dtype} is not a `torch.dtype`. Defaulting to `torch.float32`."
-            )
+        if dtype is not None and not isinstance(dtype, dict) and not isinstance(dtype, torch.dtype):
+            logger.warning(f"Passed `dtype` {dtype} is not a `torch.dtype`. Defaulting to `torch.float32`.")
+            dtype = torch.float32
 
         if low_cpu_mem_usage and not is_accelerate_available():
             low_cpu_mem_usage = False
@@ -1019,7 +1024,7 @@ def load_module(name, value):
                 init_dict=init_dict,
                 library=library,
                 max_memory=max_memory,
-                torch_dtype=torch_dtype,
+                dtype=dtype,
                 cached_folder=cached_folder,
                 force_download=force_download,
                 proxies=proxies,
@@ -1067,9 +1072,7 @@ def load_module(name, value):
             else:
                 # load sub model
                 sub_model_dtype = (
-                    torch_dtype.get(name, torch_dtype.get("default", torch.float32))
-                    if isinstance(torch_dtype, dict)
-                    else torch_dtype
+                    dtype.get(name, dtype.get("default", torch.float32)) if isinstance(dtype, dict) else dtype
                 )
                 loaded_sub_model = load_sub_model(
                     library_name=library_name,
@@ -1078,7 +1081,7 @@ def load_module(name, value):
                     pipelines=pipelines,
                     is_pipeline_module=is_pipeline_module,
                     pipeline_class=pipeline_class,
-                    torch_dtype=sub_model_dtype,
+                    dtype=sub_model_dtype,
                     provider=provider,
                     sess_options=sess_options,
                     device_map=current_device_map,
@@ -1466,7 +1469,7 @@ def enable_group_offload(
             >>> from diffusers import DiffusionPipeline
             >>> import torch
 
-            >>> pipe = DiffusionPipeline.from_pretrained("Qwen/Qwen-Image", torch_dtype=torch.bfloat16)
+            >>> pipe = DiffusionPipeline.from_pretrained("Qwen/Qwen-Image", dtype=torch.bfloat16)
 
             >>> pipe.enable_group_offload(
             ...     onload_device=torch.device("cuda"),
@@ -2055,7 +2058,7 @@ def enable_xformers_memory_efficient_attention(self, attention_op: Callable | No
         >>> from diffusers import DiffusionPipeline
         >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp
 
-        >>> pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16)
+        >>> pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1", dtype=torch.float16)
         >>> pipe = pipe.to("cuda")
         >>> pipe.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp)
         >>> # Workaround for not accepting attention shape using VAE for Flash Attention
@@ -2114,7 +2117,7 @@ def enable_attention_slicing(self, slice_size: str | int = "auto"):
 
         >>> pipe = StableDiffusionPipeline.from_pretrained(
         ...     "stable-diffusion-v1-5/stable-diffusion-v1-5",
-        ...     torch_dtype=torch.float16,
+        ...     dtype=torch.float16,
         ...     use_safetensors=True,
         ... )
 
@@ -2168,7 +2171,15 @@ def from_pipe(cls, pipeline, **kwargs):
         original_config = dict(pipeline.config)
         torch_dtype = kwargs.pop("torch_dtype", None)
         dtype = kwargs.pop("dtype", None)
-        torch_dtype = dtype if dtype is not None else (torch_dtype if torch_dtype is not None else torch.float32)
+        if torch_dtype is not None:
+            if dtype is not None:
+                raise ValueError(
+                    "You have passed both `dtype` and `torch_dtype`. Please only pass `dtype`, `torch_dtype` is deprecated."
+                )
+            deprecate("torch_dtype", "1.0.0", _TORCH_DTYPE_DEPRECATION_MESSAGE)
+            dtype = torch_dtype
+        if dtype is None:
+            dtype = torch.float32
         trust_remote_code = kwargs.pop("trust_remote_code", False)
 
         # derive the pipeline class to instantiate
@@ -2266,8 +2277,7 @@ def from_pipe(cls, pipeline, **kwargs):
             new_pipeline.register_to_config(_name_or_path=pretrained_model_name_or_path)
         new_pipeline.register_to_config(**unused_original_config)
 
-        if torch_dtype is not None:
-            new_pipeline.to(dtype=torch_dtype)
+        new_pipeline.to(dtype=dtype)
 
         return new_pipeline
 

src/diffusers/utils/__init__.py

@@ -41,7 +41,7 @@
     WEIGHTS_INDEX_NAME,
     WEIGHTS_NAME,
 )
-from .deprecation_utils import _maybe_remap_transformers_class, deprecate
+from .deprecation_utils import _TORCH_DTYPE_DEPRECATION_MESSAGE, _maybe_remap_transformers_class, deprecate
 from .doc_utils import replace_example_docstring
 from .dynamic_modules_utils import get_class_from_dynamic_module
 from .export_utils import encode_video, export_to_gif, export_to_obj, export_to_ply, export_to_video

src/diffusers/utils/deprecation_utils.py

@@ -9,6 +9,8 @@
 
 logger = logging.get_logger(__name__)
 
+_TORCH_DTYPE_DEPRECATION_MESSAGE = "Please use `dtype` instead."
+
 # Mapping for deprecated Transformers classes to their replacements
 # This is used to handle models that reference deprecated class names in their configs
 # Reference: https://github.com/huggingface/transformers/issues/40822

tests/modular_pipelines/anima/test_modular_pipeline_anima.py

@@ -155,10 +155,10 @@ class TestAnimaModularPipelineFast(ModularPipelineTesterMixin, ModularGuiderTest
     batch_params = frozenset(["prompt", "negative_prompt"])
     expected_workflow_blocks = ANIMA_TEXT2IMAGE_WORKFLOWS
 
-    def get_pipeline(self, components_manager=None, torch_dtype=torch.float32):
+    def get_pipeline(self, components_manager=None, dtype=torch.float32):
         pipe = self.pipeline_blocks_class().init_pipeline(components_manager=components_manager)
         pipe.update_components(**get_dummy_components())
-        pipe.to(dtype=torch_dtype)
+        pipe.to(dtype=dtype)
         pipe.set_progress_bar_config(disable=None)
         return pipe
 

tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py

@@ -122,8 +122,8 @@ class TestCosmos3OmniModularPipelineFast(ModularPipelineTesterMixin):
     output_name = "videos"
     expected_workflow_blocks = COSMOS3_OMNI_WORKFLOWS
 
-    def get_pipeline(self, components_manager=None, torch_dtype=torch.float32):
-        pipe = super().get_pipeline(components_manager, torch_dtype)
+    def get_pipeline(self, components_manager=None, dtype=torch.float32):
+        pipe = super().get_pipeline(components_manager, dtype)
         pipe.disable_safety_checker()
         return pipe
 
@@ -161,7 +161,7 @@ def test_save_from_pretrained(self, tmp_path):
         base_pipe.save_pretrained(str(tmp_path))
 
         loaded_pipe = ModularPipeline.from_pretrained(str(tmp_path))
-        loaded_pipe.load_components(torch_dtype=torch.float32)
+        loaded_pipe.load_components(dtype=torch.float32)
         loaded_pipe.disable_safety_checker()
         loaded_pipe.to(torch_device)
 

tests/modular_pipelines/flux/test_modular_pipeline_flux.py

@@ -98,8 +98,8 @@ class TestFluxImg2ImgModularPipelineFast(ModularPipelineTesterMixin):
     batch_params = frozenset(["prompt", "image"])
     expected_workflow_blocks = FLUX_IMAGE2IMAGE_WORKFLOWS
 
-    def get_pipeline(self, components_manager=None, torch_dtype=torch.float32):
-        pipeline = super().get_pipeline(components_manager, torch_dtype)
+    def get_pipeline(self, components_manager=None, dtype=torch.float32):
+        pipeline = super().get_pipeline(components_manager, dtype)
 
         # Override `vae_scale_factor` here as currently, `image_processor` is initialized with
         # fixed constants instead of
@@ -135,7 +135,7 @@ def test_save_from_pretrained(self, tmp_path):
 
         base_pipe.save_pretrained(str(tmp_path))
         pipe = ModularPipeline.from_pretrained(tmp_path).to(torch_device)
-        pipe.load_components(torch_dtype=torch.float32)
+        pipe.load_components(dtype=torch.float32)
         pipe.to(torch_device)
         pipe.image_processor = VaeImageProcessor(vae_scale_factor=2)
 
@@ -216,7 +216,7 @@ def test_save_from_pretrained(self, tmp_path):
 
         base_pipe.save_pretrained(str(tmp_path))
         pipe = ModularPipeline.from_pretrained(tmp_path).to(torch_device)
-        pipe.load_components(torch_dtype=torch.float32)
+        pipe.load_components(dtype=torch.float32)
         pipe.to(torch_device)
         pipe.image_processor = VaeImageProcessor(vae_scale_factor=2)