Restore pluggable email backend for task failure/retry alerts

airflow-core/src/airflow/config_templates/config.yml

@@ -2394,6 +2394,19 @@ email:
       type: string
       example: ~
       default: "airflow.utils.email.send_email_smtp"
+    email_notifier:
+      description: |
+        Dotted path to the notifier class used to deliver ``email_on_failure`` and
+        ``email_on_retry`` task alerts. The class must be a ``BaseNotifier`` subclass whose
+        constructor accepts ``to``, ``from_email``, ``subject`` and ``html_content`` keyword
+        arguments. Override it to route failure/retry alerts through a backend other than SMTP
+        (for example an SES- or SendGrid-based notifier). Both the worker task-runner path and
+        the DAG-processor callback path load the notifier from this single option, so the
+        selected backend is used consistently regardless of how the task failed.
+      version_added: 3.3.1
+      type: string
+      example: ~
+      default: "airflow.providers.smtp.notifications.smtp.SmtpNotifier"
     email_conn_id:
       description: Email connection to use
       version_added: 2.1.0

providers/common/compat/src/airflow/providers/common/compat/notifier/__init__.py

@@ -17,16 +17,70 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from airflow.providers.common.compat.version_compat import AIRFLOW_V_3_0_PLUS
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from airflow.sdk.bases.notifier import BaseNotifier
+    from airflow.sdk.definitions.context import Context
 elif AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.bases.notifier import BaseNotifier
 else:
     from airflow.notifications.basenotifier import BaseNotifier
 
 
-__all__ = ["BaseNotifier"]
+DEFAULT_EMAIL_BACKEND = "airflow.utils.email.send_email_smtp"
+
+
+class LegacyEmailBackendNotifier(BaseNotifier):
+    """
+    Adapter that exposes a legacy ``[email] email_backend`` callable as a notifier.
+
+    Before failure/retry alerts were routed through ``BaseNotifier`` subclasses, deployments
+    configured them through ``[email] email_backend`` -- a callable with the
+    ``airflow.utils.email.send_email`` signature, for example the SES or SendGrid emailers. This
+    adapter bridges the old and new mechanisms: it renders the standard email fields like any
+    notifier, then dynamically loads and calls the configured backend callable, so existing
+    ``email_backend`` setups keep working without change.
+
+    The backend is loaded lazily from config at notify time via ``conf.getimport`` rather than
+    imported statically, keeping callers free of a hard dependency on ``airflow.utils.email``
+    (which lives in ``airflow-core``).
+    """
+
+    template_fields = ("to", "from_email", "subject", "html_content")
+
+    def __init__(
+        self,
+        to: str | Iterable[str],
+        from_email: str | None = None,
+        subject: str | None = None,
+        html_content: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__()
+        self.to = to
+        self.from_email = from_email
+        self.subject = subject
+        self.html_content = html_content
+
+    def notify(self, context: Context) -> None:
+        from airflow.providers.common.compat.sdk import AirflowConfigException, conf
+
+        backend = conf.getimport("email", "email_backend", fallback=DEFAULT_EMAIL_BACKEND)
+        if backend is None:
+            raise AirflowConfigException("`[email] email_backend` is not configured")
+        conn_id = conf.get("email", "email_conn_id", fallback=None)
+        backend(
+            self.to,
+            self.subject,
+            self.html_content,
+            conn_id=conn_id,
+            from_email=self.from_email,
+        )
+
+
+__all__ = ["BaseNotifier", "LegacyEmailBackendNotifier"]

providers/common/compat/tests/unit/common/compat/test_notifier.py

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest import mock
+
+import pytest
+
+from airflow.providers.common.compat.notifier import LegacyEmailBackendNotifier
+
+
+class TestLegacyEmailBackendNotifier:
+    def test_notify_calls_configured_backend(self):
+        """notify() loads the legacy backend from config and calls it with the standard fields."""
+        from airflow.providers.common.compat.sdk import conf
+
+        backend = mock.MagicMock()
+        notifier = LegacyEmailBackendNotifier(
+            to=["a@b.com"],
+            from_email="from@x.com",
+            subject="Subject",
+            html_content="<p>body</p>",
+        )
+        with (
+            mock.patch.object(conf, "getimport", return_value=backend) as getimport,
+            mock.patch.object(conf, "get", return_value="my_conn"),
+        ):
+            notifier.notify(context={})
+
+        getimport.assert_called_once_with(
+            "email", "email_backend", fallback="airflow.utils.email.send_email_smtp"
+        )
+        backend.assert_called_once_with(
+            ["a@b.com"],
+            "Subject",
+            "<p>body</p>",
+            conn_id="my_conn",
+            from_email="from@x.com",
+        )
+
+    def test_notify_raises_when_backend_unresolvable(self):
+        """An empty/unloadable backend raises rather than silently doing nothing."""
+        from airflow.providers.common.compat.sdk import AirflowConfigException, conf
+
+        notifier = LegacyEmailBackendNotifier(to="a@b.com")
+        with mock.patch.object(conf, "getimport", return_value=None):
+            with pytest.raises(AirflowConfigException):
+                notifier.notify(context={})

task-sdk/src/airflow/sdk/execution_time/task_runner.py

@@ -71,6 +71,7 @@
 from airflow.sdk.definitions.mappedoperator import MappedOperator
 from airflow.sdk.definitions.param import process_params
 from airflow.sdk.exceptions import (
+    AirflowConfigException,
     AirflowException,
     AirflowInactiveAssetInInletOrOutletException,
     AirflowRescheduleException,
@@ -1998,20 +1999,69 @@ def _send_error_email_notification(
     error: BaseException | str | None,
     log: Logger,
 ) -> None:
-    """Send email notification for task errors using SmtpNotifier."""
-    try:
-        from airflow.providers.smtp.notifications.smtp import SmtpNotifier
-    except ImportError:
-        log.error(
-            "Failed to send task failure or retry email notification: "
-            "`apache-airflow-providers-smtp` is not installed. "
-            "Install this provider to enable email notifications."
-        )
-        return
-
+    """
+    Send email notification for task errors using the configured notifier.
+
+    The notifier class is selected in this order of precedence:
+
+    1. ``[email] email_notifier`` if explicitly set to a non-default value -- any ``BaseNotifier``
+       subclass whose constructor accepts the standard ``to``, ``from_email``, ``subject`` and
+       ``html_content`` keyword arguments (for example an SES- or SendGrid-based notifier).
+    2. Otherwise, if a legacy ``[email] email_backend`` callable is configured (a non-default
+       value with the ``airflow.utils.email.send_email`` signature), it is wrapped in
+       :class:`~airflow.sdk.execution_time.email_backend_compat.LegacyEmailBackendNotifier` so
+       existing ``email_backend`` deployments keep working.
+    3. Otherwise, the default :class:`~airflow.providers.smtp.notifications.smtp.SmtpNotifier`.
+
+    Both the worker task-runner path (:func:`finalize`) and the DAG-processor callback path
+    (``_execute_email_callbacks``) funnel through this function, so the resolved notifier is used
+    consistently regardless of how the task failed.
+    """
     if not task.email:
         return
 
+    default_notifier = "airflow.providers.smtp.notifications.smtp.SmtpNotifier"
+    default_backend = "airflow.utils.email.send_email_smtp"
+    notifier_path = conf.get("email", "email_notifier", fallback=default_notifier)
+    email_backend = conf.get("email", "email_backend", fallback=default_backend)
+
+    if notifier_path == default_notifier and email_backend and email_backend != default_backend:
+        # No explicit notifier override, but a custom legacy email_backend is configured -- adapt
+        # it so SES/SendGrid/custom backends keep delivering failure and retry alerts.
+        try:
+            from airflow.providers.common.compat.notifier import LegacyEmailBackendNotifier
+        except ImportError:
+            log.error(
+                "Failed to send task failure or retry email notification: a legacy "
+                "`[email] email_backend` (%r) is configured but the compat shim is unavailable. "
+                "Upgrade `apache-airflow-providers-common-compat`, or migrate to a "
+                "`[email] email_notifier`.",
+                email_backend,
+            )
+            return
+        notifier_class: type = LegacyEmailBackendNotifier
+        notifier_path = f"legacy email_backend {email_backend!r}"
+    else:
+        try:
+            notifier_class = conf.getimport("email", "email_notifier", fallback=default_notifier)
+        except AirflowConfigException:
+            # getimport wraps ImportError, so a missing provider (the default SmtpNotifier lives
+            # in apache-airflow-providers-smtp) or a bad dotted path both land here.
+            log.error(
+                "Failed to send task failure or retry email notification: could not load the "
+                "configured email notifier %r. If you rely on the default SMTP notifier, install "
+                "`apache-airflow-providers-smtp`; otherwise check the `[email] email_notifier` config.",
+                notifier_path,
+            )
+            return
+        if notifier_class is None:
+            log.error(
+                "Failed to send task failure or retry email notification: `[email] email_notifier` "
+                "is empty. Set it to a BaseNotifier subclass (default: %r).",
+                default_notifier,
+            )
+            return
+
     subject_template_file = conf.get("email", "subject_template", fallback=None)
 
     # Read the template file if configured
@@ -2054,15 +2104,15 @@ def _send_error_email_notification(
         return
 
     try:
-        notifier = SmtpNotifier(
+        notifier = notifier_class(
             to=to_emails,
             subject=subject,
             html_content=html_content,
             from_email=conf.get("email", "from_email", fallback="airflow@airflow"),
         )
         notifier(email_context)
     except Exception:
-        log.exception("Failed to send email notification")
+        log.exception("Failed to send email notification via %s", notifier_path)
 
 
 @detail_span("task.execute")