Fetch and use compressed manifest.json

metadata_manager/firmware_server/client.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import lzma
 import os
 from dataclasses import dataclass
 from datetime import datetime, timezone
@@ -34,7 +35,7 @@ def from_dict(cls, data: dict) -> "_CacheMeta":
 
 
 class ManifestClient:
-    """Fetch and cache the ArduPilot firmware manifest.json file."""
+    """Fetch and cache the ArduPilot firmware manifest.json.xz file."""
 
     def __init__(
         self,
@@ -81,7 +82,8 @@ def fetch_raw(self) -> bytes:
                 )
             )
 
-        raw = response.content
+        wire_bytes = response.content
+        raw = lzma.decompress(wire_bytes)
         self._write_cache(
             raw,
             _CacheMeta(
@@ -90,7 +92,11 @@ def fetch_raw(self) -> bytes:
                 fetched_at=self._now_iso(),
             ),
         )
-        self.logger.info("Downloaded manifest (%d bytes)", len(raw))
+        self.logger.info(
+            "Downloaded manifest (%d wire bytes, %d decompressed bytes)",
+            len(wire_bytes),
+            len(raw),
+        )
         return raw
 
     def fetch(self) -> dict:

tests/metadata_manager/test_firmware_server.py

@@ -1,4 +1,5 @@
 import json
+import lzma
 from pathlib import Path
 from unittest.mock import Mock, patch
 
@@ -8,6 +9,8 @@
 
 FIXTURES_DIR = Path(__file__).parent / "fixtures"
 SAMPLE_MANIFEST = json.loads((FIXTURES_DIR / "manifest_sample.json").read_text())
+SAMPLE_JSON_BYTES = json.dumps(SAMPLE_MANIFEST).encode("utf-8")
+MANIFEST_XZ_URL = "https://firmware.ardupilot.org/manifest.json.xz"
 
 
 class TestManifestIndex:
@@ -39,7 +42,7 @@ def test_builds_releases_for_copter_heli_and_tracker(self):
 class TestManifestClientCache:
     def test_uses_cache_on_304(self, tmp_path):
         client = ManifestClient(
-            url="https://example.com/manifest.json",
+            url=MANIFEST_XZ_URL,
             cache_dir=str(tmp_path),
         )
         client._write_cache(
@@ -48,7 +51,49 @@ def test_uses_cache_on_304(self, tmp_path):
         )
 
         response = Mock(status_code=304, headers={}, content=b"")
-        with patch("metadata_manager.firmware_server.client.requests.get", return_value=response):
+        with patch(
+            "metadata_manager.firmware_server.client.requests.get",
+            return_value=response,
+        ) as mock_get:
             raw = client.fetch_raw()
 
         assert raw == b'{"format-version":"1.0.0","firmware":[]}'
+        mock_get.assert_called_once_with(
+            MANIFEST_XZ_URL,
+            headers={
+                "User-Agent": "CustomBuild/1.0",
+                "If-None-Match": '"abc"',
+                "If-Modified-Since": "Mon, 01 Jan 2024 00:00:00 GMT",
+            },
+            timeout=120,
+        )
+
+    def test_download_decompresses_xz_manifest(self, tmp_path):
+        client = ManifestClient(
+            url=MANIFEST_XZ_URL,
+            cache_dir=str(tmp_path),
+        )
+        compressed = lzma.compress(SAMPLE_JSON_BYTES)
+        response = Mock(
+            status_code=200,
+            content=compressed,
+            headers={
+                "ETag": '"etag123"',
+                "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT",
+            },
+        )
+        response.raise_for_status = Mock()
+
+        with patch(
+            "metadata_manager.firmware_server.client.requests.get",
+            return_value=response,
+        ):
+            result = client.fetch()
+
+        assert result == SAMPLE_MANIFEST
+        assert client.cache_path.read_bytes() == SAMPLE_JSON_BYTES
+        meta = _CacheMeta.from_dict(
+            json.loads(client.meta_path.read_text(encoding="utf-8"))
+        )
+        assert meta.etag == '"etag123"'
+        assert meta.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"

tests/web/test_config.py

@@ -23,6 +23,7 @@ def test_default_settings(self):
             assert settings.log_level == "INFO"
             assert settings.ap_git_url == "https://github.com/ardupilot/ardupilot.git"
             assert settings.enable_inbuilt_builder is True
+            assert settings.ap_firmware_manifest_url.endswith(".xz")
 
     def test_env_var_overrides(self):
         """Test that environment variables override default settings."""

web/core/config.py

@@ -97,7 +97,7 @@ def ap_firmware_manifest_url(self) -> str:
         """URL for the ArduPilot firmware manifest."""
         return os.getenv(
             'CBS_AP_FIRMWARE_MANIFEST_URL',
-            'https://firmware.ardupilot.org/manifest.json',
+            'https://firmware.ardupilot.org/manifest.json.xz',
         )
 
     @property