perf: Replace tldextract with a built-in Public Suffix List lookup

.github/workflows/update_playwright_version.yaml

@@ -4,9 +4,9 @@ on:
   # Runs when manually triggered from the GitHub UI.
   workflow_dispatch:
 
-  # Runs every day at 04:00 UTC.
+  # Runs every workday (Monday to Friday) at 04:00 UTC.
   schedule:
-    - cron: '0 4 * * *'
+    - cron: '0 4 * * 1-5'
 
 concurrency:
   group: update-playwright-version

.github/workflows/update_public_suffix_list.yaml

@@ -0,0 +1,97 @@
+name: Update Public Suffix List
+
+on:
+  # Runs when manually triggered from the GitHub UI.
+  workflow_dispatch:
+
+  # Runs every workday (Monday to Friday) at 04:00 UTC.
+  schedule:
+    - cron: '0 4 * * 1-5'
+
+concurrency:
+  group: update-public-suffix-list
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  BRANCH_NAME: ci/update-public-suffix-list
+
+jobs:
+  update-public-suffix-list:
+    name: Update Public Suffix List
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          fetch-depth: 0
+
+      # Continue an already-open update branch so the refresh lands on the existing PR (preserving
+      # its history, comments and approvals) instead of opening a fresh PR on every run. When no
+      # update branch exists yet, stay on the default branch so a new one is created below.
+      - name: Switch to the update branch if it exists
+        run: |
+          if git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1; then
+            git fetch origin "$BRANCH_NAME"
+            git checkout -B "$BRANCH_NAME" FETCH_HEAD
+          fi
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.14'
+
+      - name: Update Public Suffix List
+        run: python scripts/update_public_suffix_list.py
+
+      - name: Detect changes
+        id: changes
+        run: |
+          if git diff --quiet; then
+            echo "has-changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has-changes=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Commit and push changes
+        id: commit-and-push
+        if: steps.changes.outputs.has-changes == 'true'
+        uses: apify/actions/signed-commit@v1.6.2
+        with:
+          message: 'chore: update Public Suffix List snapshot'
+          github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          branch: ${{ env.BRANCH_NAME }}
+          create-branch: 'true'
+
+      - name: Create or update Pull Request
+        if: steps.commit-and-push.outputs.committed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+        run: |
+          PR_BODY="Automated refresh of the ICANN section of the Public Suffix List bundled for the \`same-domain\` enqueue strategy.
+
+          > Generated by the [Update Public Suffix List](https://github.com/apify/crawlee-python/actions/workflows/update_public_suffix_list.yaml) workflow."
+
+          PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --json number --jq '.[0].number // empty')
+          if [ -z "$PR_NUMBER" ]; then
+            echo "Creating new PR"
+            gh pr create \
+              --title "chore: update Public Suffix List snapshot" \
+              --body "$PR_BODY" \
+              --base master \
+              --head "$BRANCH_NAME"
+            PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --json number --jq '.[0].number // empty')
+          else
+            echo "PR #$PR_NUMBER already exists for branch $BRANCH_NAME, updating it"
+            gh pr edit "$PR_NUMBER" --body "$PR_BODY"
+          fi
+
+          # Enable auto-merge so the PR merges automatically once CI passes. If CI fails, the PR
+          # stays open for manual review.
+          echo "Enabling auto-merge for PR #$PR_NUMBER"
+          gh pr merge "$PR_NUMBER" --auto --squash --delete-branch \
+            || echo "::warning::Failed to enable auto-merge. The PR remains open for manual review."

pyproject.toml

@@ -44,7 +44,6 @@ dependencies = [
     "psutil>=6.0.0",
     "pydantic-settings>=2.12.0",
     "pydantic>=2.11.0",
-    "tldextract>=5.3.0",
     "typing-extensions>=4.10.0",
     "yarl>=1.18.0",
 ]
@@ -126,6 +125,7 @@ dev = [
     "apify_client<3.0.0", # For e2e tests.
     "build<2.0.0", # For e2e tests.
     "fakeredis[probabilistic,json,lua]<3.0.0",
+    "filelock<5.0.0", # For e2e tests.
     "poethepoet<1.0.0",
     "pre-commit<5.0.0",
     "proxy-py<3.0.0",

scripts/update_public_suffix_list.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Refresh the bundled snapshot of the Public Suffix List.
+
+The `same-domain` enqueue strategy matches only against the ICANN section of the list, so the snapshot keeps the
+license notice and the ICANN rules without comments. It then changes only when the ICANN rules do.
+
+Single-purpose: run with no arguments from anywhere in the repository.
+"""
+
+from __future__ import annotations
+
+import urllib.request
+from itertools import takewhile
+from pathlib import Path
+
+SNAPSHOT = Path(__file__).resolve().parent.parent / 'src/crawlee/_utils/resources/public_suffix_list.dat'
+LIST_URL = 'https://publicsuffix.org/list/public_suffix_list.dat'
+
+ICANN_BEGIN = '// ===BEGIN ICANN DOMAINS==='
+ICANN_END = '// ===END ICANN DOMAINS==='
+
+
+def main() -> None:
+    with urllib.request.urlopen(LIST_URL, timeout=30) as response:
+        lines = [line.strip() for line in response.read().decode('utf-8').splitlines()]
+
+    # The leading comment block is the MPL 2.0 license notice, which has to ship with the list.
+    license_notice = list(takewhile(lambda line: line.startswith('//'), lines))
+    begin, end = lines.index(ICANN_BEGIN), lines.index(ICANN_END)
+    rules = [line for line in lines[begin + 1 : end] if line and not line.startswith('//')]
+
+    SNAPSHOT.write_text(
+        '\n'.join(
+            [
+                *license_notice,
+                '',
+                f'// The ICANN section of {LIST_URL}, generated by scripts/update_public_suffix_list.py.',
+                '',
+                ICANN_BEGIN,
+                *rules,
+                ICANN_END,
+                '',
+            ]
+        ),
+        encoding='utf-8',
+    )
+
+
+if __name__ == '__main__':
+    main()

src/crawlee/_utils/public_suffix.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from functools import cached_property
+from importlib.resources import files
+from ipaddress import ip_address
+from logging import getLogger
+from typing import NamedTuple
+
+import impit
+
+from crawlee._utils.web import is_status_code_successful
+
+logger = getLogger(__name__)
+
+PUBLIC_SUFFIX_LIST_URL = 'https://publicsuffix.org/list/public_suffix_list.dat'
+
+PUBLIC_SUFFIX_LIST_SNAPSHOT = files('crawlee._utils') / 'resources' / 'public_suffix_list.dat'
+
+_ICANN_BEGIN = '// ===BEGIN ICANN DOMAINS==='
+_ICANN_END = '// ===END ICANN DOMAINS==='
+
+# The download is synchronous and blocks the event loop, so it gets a short timeout.
+_DOWNLOAD_TIMEOUT_SECS = 5
+
+
+class _Rules(NamedTuple):
+    """Rules of the ICANN section of the Public Suffix List, split by type."""
+
+    suffixes: frozenset[str]
+    """Plain rules, e.g. `co.uk`."""
+
+    wildcards: frozenset[str]
+    """Wildcard rules without the leading `*.`, e.g. `kawasaki.jp` for `*.kawasaki.jp`."""
+
+    exceptions: frozenset[str]
+    """Exception rules without the leading `!`, e.g. `city.kawasaki.jp`."""
+
+
+class PublicSuffixList:
+    """Registrable domain lookup based on the ICANN section of the Public Suffix List.
+
+    The rules are downloaded on first use, falling back to the bundled snapshot if the download fails. After that,
+    they don't change.
+    """
+
+    def get_registrable_domain(self, host: str) -> str | None:
+        """Return the registrable domain of `host`, or `None` for IP addresses and public suffixes."""
+        host = host.rstrip('.')
+        if self._is_ip_address(host):
+            return None
+
+        labels = host.split('.')
+        for i in range(len(labels)):
+            suffix = '.'.join(labels[i:])
+            if suffix in self._rules.exceptions:
+                return suffix
+            if suffix in self._rules.suffixes or suffix.partition('.')[2] in self._rules.wildcards:
+                return '.'.join(labels[i - 1 :]) if i > 0 else None
+
+        # No rule matched, so the default `*` rule makes the last label the public suffix.
+        return '.'.join(labels[-2:]) if len(labels) > 1 else None
+
+    @cached_property
+    def _rules(self) -> _Rules:
+        try:
+            return self._parse_rules(self._download_list())
+        except Exception as exc:
+            logger.warning(f'Failed to download the Public Suffix List, using the bundled snapshot: {exc!r}')
+            return self._parse_rules(PUBLIC_SUFFIX_LIST_SNAPSHOT.read_text(encoding='utf-8'))
+
+    @staticmethod
+    def _download_list() -> str:
+        response = impit.get(PUBLIC_SUFFIX_LIST_URL, timeout=_DOWNLOAD_TIMEOUT_SECS, follow_redirects=True)
+        if not is_status_code_successful(response.status_code):
+            raise RuntimeError(f'Unexpected HTTP status code {response.status_code}.')
+
+        return response.content.decode('utf-8')
+
+    @staticmethod
+    def _parse_rules(text: str) -> _Rules:
+        start = text.find(_ICANN_BEGIN)
+        end = text.find(_ICANN_END)
+        if start == -1 or end < start:
+            raise ValueError('The ICANN section of the Public Suffix List is missing.')
+
+        suffixes: set[str] = set()
+        wildcards: set[str] = set()
+        exceptions: set[str] = set()
+        for line in text[start:end].splitlines():
+            rule = line.strip()
+            if not rule or rule.startswith('//'):
+                continue
+            if rule.startswith('!'):
+                exceptions.add(rule[1:])
+            elif rule.startswith('*.'):
+                wildcards.add(rule[2:])
+            else:
+                suffixes.add(rule)
+
+        return _Rules(frozenset(suffixes), frozenset(wildcards), frozenset(exceptions))
+
+    @staticmethod
+    def _is_ip_address(host: str) -> bool:
+        try:
+            ip_address(host)
+        except ValueError:
+            return False
+        return True
+
+
+public_suffix_list = PublicSuffixList()
+"""The process-wide `PublicSuffixList` instance."""