diff --git a/metadata_manager/firmware_server/client.py b/metadata_manager/firmware_server/client.py index 0c68a22..b7d953c 100644 --- a/metadata_manager/firmware_server/client.py +++ b/metadata_manager/firmware_server/client.py @@ -1,5 +1,6 @@ import json import logging +import lzma import os from dataclasses import dataclass from datetime import datetime, timezone @@ -34,7 +35,7 @@ def from_dict(cls, data: dict) -> "_CacheMeta": class ManifestClient: - """Fetch and cache the ArduPilot firmware manifest.json file.""" + """Fetch and cache the ArduPilot firmware manifest.json.xz file.""" def __init__( self, @@ -81,7 +82,8 @@ def fetch_raw(self) -> bytes: ) ) - raw = response.content + wire_bytes = response.content + raw = lzma.decompress(wire_bytes) self._write_cache( raw, _CacheMeta( @@ -90,7 +92,11 @@ def fetch_raw(self) -> bytes: fetched_at=self._now_iso(), ), ) - self.logger.info("Downloaded manifest (%d bytes)", len(raw)) + self.logger.info( + "Downloaded manifest (%d wire bytes, %d decompressed bytes)", + len(wire_bytes), + len(raw), + ) return raw def fetch(self) -> dict: diff --git a/tests/metadata_manager/test_firmware_server.py b/tests/metadata_manager/test_firmware_server.py index 518720a..545ee52 100644 --- a/tests/metadata_manager/test_firmware_server.py +++ b/tests/metadata_manager/test_firmware_server.py @@ -1,4 +1,5 @@ import json +import lzma from pathlib import Path from unittest.mock import Mock, patch @@ -8,6 +9,8 @@ FIXTURES_DIR = Path(__file__).parent / "fixtures" SAMPLE_MANIFEST = json.loads((FIXTURES_DIR / "manifest_sample.json").read_text()) +SAMPLE_JSON_BYTES = json.dumps(SAMPLE_MANIFEST).encode("utf-8") +MANIFEST_XZ_URL = "https://firmware.ardupilot.org/manifest.json.xz" class TestManifestIndex: @@ -39,7 +42,7 @@ def test_builds_releases_for_copter_heli_and_tracker(self): class TestManifestClientCache: def test_uses_cache_on_304(self, tmp_path): client = ManifestClient( - url="https://example.com/manifest.json", + url=MANIFEST_XZ_URL, cache_dir=str(tmp_path), ) client._write_cache( @@ -48,7 +51,49 @@ def test_uses_cache_on_304(self, tmp_path): ) response = Mock(status_code=304, headers={}, content=b"") - with patch("metadata_manager.firmware_server.client.requests.get", return_value=response): + with patch( + "metadata_manager.firmware_server.client.requests.get", + return_value=response, + ) as mock_get: raw = client.fetch_raw() assert raw == b'{"format-version":"1.0.0","firmware":[]}' + mock_get.assert_called_once_with( + MANIFEST_XZ_URL, + headers={ + "User-Agent": "CustomBuild/1.0", + "If-None-Match": '"abc"', + "If-Modified-Since": "Mon, 01 Jan 2024 00:00:00 GMT", + }, + timeout=120, + ) + + def test_download_decompresses_xz_manifest(self, tmp_path): + client = ManifestClient( + url=MANIFEST_XZ_URL, + cache_dir=str(tmp_path), + ) + compressed = lzma.compress(SAMPLE_JSON_BYTES) + response = Mock( + status_code=200, + content=compressed, + headers={ + "ETag": '"etag123"', + "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT", + }, + ) + response.raise_for_status = Mock() + + with patch( + "metadata_manager.firmware_server.client.requests.get", + return_value=response, + ): + result = client.fetch() + + assert result == SAMPLE_MANIFEST + assert client.cache_path.read_bytes() == SAMPLE_JSON_BYTES + meta = _CacheMeta.from_dict( + json.loads(client.meta_path.read_text(encoding="utf-8")) + ) + assert meta.etag == '"etag123"' + assert meta.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT" diff --git a/tests/web/test_config.py b/tests/web/test_config.py index c460475..123a3f3 100644 --- a/tests/web/test_config.py +++ b/tests/web/test_config.py @@ -23,6 +23,7 @@ def test_default_settings(self): assert settings.log_level == "INFO" assert settings.ap_git_url == "https://github.com/ardupilot/ardupilot.git" assert settings.enable_inbuilt_builder is True + assert settings.ap_firmware_manifest_url.endswith(".xz") def test_env_var_overrides(self): """Test that environment variables override default settings.""" diff --git a/web/core/config.py b/web/core/config.py index ee09cee..0c8f0f4 100644 --- a/web/core/config.py +++ b/web/core/config.py @@ -97,7 +97,7 @@ def ap_firmware_manifest_url(self) -> str: """URL for the ArduPilot firmware manifest.""" return os.getenv( 'CBS_AP_FIRMWARE_MANIFEST_URL', - 'https://firmware.ardupilot.org/manifest.json', + 'https://firmware.ardupilot.org/manifest.json.xz', ) @property