Fix download filename truncation for multibyte characters

CHANGELOG.md

@@ -3,6 +3,10 @@
 This document records all notable changes to [HTTPie](https://httpie.io).
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+## Unreleased
+
+- Fix downloads with multibyte filenames exceeding the filesystem byte limit, preserving extensions and collision suffixes.
+
 ## [3.2.4](https://github.com/httpie/cli/compare/3.2.3...3.2.4) (2024-11-01)
 
 - Fix default certs loading and unpin `requests`. ([#1596](https://github.com/httpie/cli/issues/1596))

httpie/downloads.py

@@ -123,15 +123,28 @@ def filename_from_url(url: str, content_type: Optional[str]) -> str:
     return fn
 
 
+def filename_length(filename: str) -> int:
+    # PC_NAME_MAX counts encoded bytes on POSIX. Keep the character-count
+    # fallback used on Windows, where pathconf is unavailable.
+    return len(filename) if os.name == 'nt' else len(os.fsencode(filename))
+
+
 def trim_filename(filename: str, max_len: int) -> str:
-    if len(filename) > max_len:
-        trim_by = len(filename) - max_len
-        name, ext = os.path.splitext(filename)
-        if trim_by >= len(name):
-            filename = filename[:-trim_by]
-        else:
-            filename = name[:-trim_by] + ext
-    return filename
+    length = filename_length(filename)
+    if length <= max_len:
+        return filename
+
+    name, ext = os.path.splitext(filename)
+    if filename_length(ext) >= max_len:
+        name, ext = filename, ''
+    remaining = filename_length(name) + filename_length(ext)
+    trim_by = 0
+    for character in reversed(name):
+        if remaining <= max_len:
+            break
+        remaining -= filename_length(character)
+        trim_by += 1
+    return name[:-trim_by] + ext
 
 
 def get_filename_max_length(directory: str) -> int:
@@ -143,7 +156,7 @@ def get_filename_max_length(directory: str) -> int:
 
 def trim_filename_if_needed(filename: str, directory='.', extra=0) -> str:
     max_len = get_filename_max_length(directory) - extra
-    if len(filename) > max_len:
+    if filename_length(filename) > max_len:
         filename = trim_filename(filename, max_len)
     return filename
 

tests/test_downloads.py

@@ -119,6 +119,35 @@ def exists(filename):
         actual = get_unique_filename(orig_name, attempts(unique_on_attempt))
         assert expected == actual
 
+    @pytest.mark.skipif(os.name == 'nt', reason='POSIX filename limits count bytes')
+    @pytest.mark.parametrize('filename, expected', [
+        ('é' * 5, 'é' * 5),
+        ('é' * 6, 'é' * 5),
+        ('猫' * 4 + '.txt', '猫' * 2 + '.txt'),
+        ('猫' * 4, '猫' * 3),
+        ('x.' + '猫' * 4, 'x.' + '猫' * 2),
+        ('é' * 4 + '.猫', 'é' * 3 + '.猫'),
+    ])
+    @mock.patch('httpie.downloads.get_filename_max_length', return_value=10)
+    def test_unique_filename_multibyte(self, max_length, filename, expected):
+        assert get_unique_filename(filename, exists=lambda _: False) == expected
+
+    @pytest.mark.skipif(os.name == 'nt', reason='POSIX filename limits count bytes')
+    def test_multibyte_download_filename_can_be_created(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        limit = os.pathconf(tmp_path, 'PC_NAME_MAX')
+        filename = '猫' * (limit // 3) + '.txt'
+        response = Response('http://example.org/', headers={
+            'Content-Disposition': "attachment; filename*=UTF-8''" + requests.utils.quote(filename),
+        })
+        for suffix in ('', '-1'):
+            with Downloader._get_output_file_from_response(response.url, response) as output:
+                assert output.name.endswith('.txt' + suffix)
+                assert len(os.fsencode(output.name)) <= limit
+                assert '猫' in output.name
+                output.write(b'downloaded')
+            assert (tmp_path / output.name).read_bytes() == b'downloaded'
+
 
 class TestDownloads:
 

tests/test_windows_filename_fallback.py

@@ -0,0 +1,38 @@
+import os
+
+import pytest
+
+from httpie import downloads
+
+
+pytestmark = pytest.mark.skipif(
+    os.name != 'nt', reason='Windows character-count filename fallback'
+)
+
+
+@pytest.mark.parametrize('filename, expected', [
+    ('\u732b' * 4 + '.txt', '\u732b' * 4 + '.txt'),
+    ('\u732b' * 8 + '.txt', '\u732b' * 6 + '.txt'),
+    ('x.' + '\u732b' * 12, 'x.' + '\u732b' * 8),
+    ('\u00e9' * 8 + '.\u732b', '\u00e9' * 8 + '.\u732b'),
+])
+def test_windows_filename_character_budget(monkeypatch, filename, expected):
+    monkeypatch.setattr(downloads, 'get_filename_max_length', lambda _: 10)
+    assert downloads.get_unique_filename(filename, exists=lambda _: False) == expected
+
+
+def test_windows_filename_collision_can_be_written(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(downloads, 'get_filename_max_length', lambda _: 10)
+    original = '\u732b' * 6 + '.txt'
+    expected = ('\u732b' * 6 + '.txt', '\u732b' * 4 + '.txt-1')
+    for attempt, expected_name in enumerate(expected):
+        filename = downloads.get_unique_filename(original)
+        assert filename == expected_name
+        assert len(filename) == 10
+        payload = str(attempt).encode('ascii')
+        path = tmp_path / filename
+        with path.open('xb') as output:
+            output.write(payload)
+        assert path.read_bytes() == payload
+    assert {path.name for path in tmp_path.iterdir()} == set(expected)