diff --git a/examples/remotes.json.sample b/examples/remotes.json.sample index 992ca00..bfdddfc 100644 --- a/examples/remotes.json.sample +++ b/examples/remotes.json.sample @@ -9,7 +9,6 @@ { "release_type": "latest", "version_number": "4.6.0", - "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/latest", "commit_reference": "refs/heads/master" } ] @@ -42,13 +41,11 @@ { "release_type": "latest", "version_number": "4.6.0", - "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/latest", "commit_reference": "202cc6ae9d326a172be5ec1120d79595a6ddae36" }, { "release_type": "stable", "version_number": "4.3.0", - "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/stable-4.3.0", "commit_reference": "93448b71380c417644c9082b7b23e80fb982b626" } ] diff --git a/metadata_manager/__init__.py b/metadata_manager/__init__.py index 6dc2ff5..2805d99 100644 --- a/metadata_manager/__init__.py +++ b/metadata_manager/__init__.py @@ -1,5 +1,5 @@ from .ap_src_meta_fetcher import APSourceMetadataFetcher -from .firmware_server import ManifestJSON, ManifestFetchError, ReleaseRecord +from .firmware_server import BoardArtifact, FeaturesTxtClient, ManifestJSON, ManifestFetchError, ReleaseRecord from .firmware_server.client import ManifestClient from .firmware_server.index import ManifestIndex from .vehicles_manager import DEFAULT_VEHICLES, Vehicle, VehiclesManager @@ -18,6 +18,8 @@ __all__ = [ "APSourceMetadataFetcher", + "BoardArtifact", + "FeaturesTxtClient", "DEFAULT_VEHICLES", "DEFAULT_WHITELISTED_FORK_REMOTES", "ForkRemoteSpec", diff --git a/metadata_manager/ap_src_meta_fetcher.py b/metadata_manager/ap_src_meta_fetcher.py index 084ab34..ea0a604 100644 --- a/metadata_manager/ap_src_meta_fetcher.py +++ b/metadata_manager/ap_src_meta_fetcher.py @@ -478,79 +478,6 @@ def get_build_options_at_commit(self, remote: str, ) return build_options - def get_board_defaults_from_fw_server( - self, - artifacts_url: str, - board_id: str, - vehicle_id: str = None, - ) -> dict: - """ - Fetch board defaults from firmware.ardupilot.org features.txt. - - The features.txt file contains lines like: - - FEATURE_NAME (enabled features) - - !FEATURE_NAME (disabled features) - - Parameters: - artifacts_url (str): Base URL for build artifacts for a version. - board_id (str): Board identifier - vehicle_id (str): Vehicle identifier - (for special handling like Heli) - - Returns: - dict: Dictionary mapping feature define to state - (1 for enabled, 0 for disabled), or None if fetch fails - """ - import requests - - # Heli builds are stored under a separate folder - artifacts_subdir = board_id - if vehicle_id == "heli": - artifacts_subdir += "-heli" - - features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt" - - try: - response = requests.get(features_txt_url, timeout=30) - response.raise_for_status() - - feature_states = {} - enabled_count = 0 - disabled_count = 0 - - for line in response.text.splitlines(): - line = line.strip() - - # Skip empty lines and comments - if not line or line.startswith('#'): - continue - - # Check if feature is disabled (prefixed with !) - if line.startswith('!'): - feature_name = line[1:].strip() - if feature_name: - feature_states[feature_name] = 0 - disabled_count += 1 - else: - # Enabled feature - if line: - feature_states[line] = 1 - enabled_count += 1 - - self.logger.info( - f"Fetched board defaults from firmware server: " - f"{enabled_count} enabled, " - f"{disabled_count} disabled" - ) - - return feature_states - - except requests.RequestException as e: - self.logger.warning( - f"Failed to fetch board defaults from {features_txt_url}: {e}" - ) - return None - @staticmethod def get_singleton(): return APSourceMetadataFetcher.__singleton diff --git a/metadata_manager/firmware_server/__init__.py b/metadata_manager/firmware_server/__init__.py index 59c270d..2cbfd34 100644 --- a/metadata_manager/firmware_server/__init__.py +++ b/metadata_manager/firmware_server/__init__.py @@ -1,8 +1,11 @@ +from .features import FeaturesTxtClient from .manifest import ManifestJSON -from .models import ReleaseRecord +from .models import BoardArtifact, ReleaseRecord from .exceptions import ManifestFetchError __all__ = [ + "BoardArtifact", + "FeaturesTxtClient", "ManifestJSON", "ManifestFetchError", "ReleaseRecord", diff --git a/metadata_manager/firmware_server/features.py b/metadata_manager/firmware_server/features.py new file mode 100644 index 0000000..2a7192d --- /dev/null +++ b/metadata_manager/firmware_server/features.py @@ -0,0 +1,117 @@ +import logging + +import dill +import redis +import requests + +FEATURES_CACHE_TTL_SEC = 86400 + + +def parse_features_txt(text: str) -> dict: + """Parse features.txt content into a define to state mapping.""" + feature_states = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if line[0] == "!": + feature_name, state = line[1:], 0 + else: + feature_name, state = line, 1 + feature_states[feature_name] = state + return feature_states + + +class FeaturesTxtClient: + """Fetch and cache board default features from firmware-server features.txt.""" + + def __init__( + self, + redis_host: str = "localhost", + redis_port: str = "6379", + caching_enabled: bool = True, + timeout: int = 30, + user_agent: str = "CustomBuild/1.0", + ): + self.caching_enabled = caching_enabled + self.timeout = timeout + self.user_agent = user_agent + self.logger = logging.getLogger(__name__) + self._cache_key_prefix = "features-" + + if self.caching_enabled: + self._redis_client = redis.Redis( + host=redis_host, + port=redis_port, + decode_responses=False, + ) + + def _cache_key(self, url: str) -> str: + return self._cache_key_prefix + url + + def _get_from_cache(self, url: str) -> tuple[str | None, dict | None]: + if not self.caching_enabled: + return None, None + value = self._redis_client.get(self._cache_key(url)) + if value is None: + return None, None + entry = dill.loads(value) + if not isinstance(entry, dict): + return None, None + etag = entry.get("etag") + defaults = entry.get("defaults") + if not etag or defaults is None: + return None, None + return etag, defaults + + def _store_in_cache(self, url: str, etag: str, defaults: dict) -> None: + if not self.caching_enabled: + return + self._redis_client.set( + name=self._cache_key(url), + value=dill.dumps({"etag": etag, "defaults": defaults}), + ex=FEATURES_CACHE_TTL_SEC, + ) + + def get_defaults(self, url: str) -> dict | None: + """ + Fetch and parse features.txt for the given URL. + + Returns a mapping of feature define to state (1 enabled, 0 disabled), + or None if the fetch fails. + """ + cached_etag, cached_defaults = self._get_from_cache(url) + headers = { + "User-Agent": self.user_agent, + "Accept-Encoding": "identity", + } + if cached_etag: + headers["If-None-Match"] = cached_etag + + try: + response = requests.get( + url, + headers=headers, + timeout=self.timeout, + ) + if response.status_code == 304: + return cached_defaults + response.raise_for_status() + defaults = parse_features_txt(response.text) + enabled_count = sum(1 for state in defaults.values() if state) + disabled_count = len(defaults) - enabled_count + self.logger.info( + "Fetched board defaults from firmware server: " + "%d enabled, %d disabled", + enabled_count, + disabled_count, + ) + etag = response.headers.get("ETag") + if etag: + self._store_in_cache(url, etag, defaults) + return defaults + except requests.RequestException as exc: + self.logger.warning( + "Failed to fetch board defaults from %s: %s", url, exc + ) + return None diff --git a/metadata_manager/firmware_server/index.py b/metadata_manager/firmware_server/index.py index 109780f..ea8a221 100644 --- a/metadata_manager/firmware_server/index.py +++ b/metadata_manager/firmware_server/index.py @@ -2,10 +2,43 @@ import re from collections import defaultdict from typing import Optional +from urllib.parse import urlparse from packaging.version import InvalidVersion, Version -from .models import ReleaseRecord +from .models import BoardArtifact, ReleaseRecord + +FIRMWARE_SERVER_BASE = "https://firmware.ardupilot.org" + +# CBS vehicle id -> firmware.ardupilot.org top-level directory. +FIRMWARE_SERVER_DIR_BY_VEHICLE_ID = { + "copter": "Copter", + "plane": "Plane", + "rover": "Rover", + "sub": "Sub", + "heli": "Copter", + "blimp": "Blimp", + "tracker": "AntennaTracker", + "ap-periph": "AP_Periph", +} + + +def firmware_server_dir(vehicle_id: str) -> str: + try: + return FIRMWARE_SERVER_DIR_BY_VEHICLE_ID[vehicle_id] + except KeyError as exc: + raise ValueError(f"Unknown vehicle id: {vehicle_id}") from exc + + +def latest_features_txt_url(vehicle_id: str, board_id: str) -> str: + """Hardcoded firmware-server latest features.txt URL for tag builds.""" + board_subdir = board_id + ("-heli" if vehicle_id == "heli" else "") + return ( + f"{FIRMWARE_SERVER_BASE}/" + f"{firmware_server_dir(vehicle_id)}/latest/" + f"{board_subdir}/features.txt" + ) + # Minimum version for a vehicle to expose from manifest entries. MIN_VERSION_BY_VEHICLE_ID = { @@ -80,9 +113,52 @@ def parse_artifacts_base_url(url: str) -> Optional[str]: return None +def manifest_platform_key(platform: str, vehicle_id: str) -> str: + """Normalize manifest platform field to the board id used for lookups.""" + if vehicle_id == "heli" and platform.endswith("-heli"): + return platform[:-5] + return platform + + +def _artifact_name_from_url(url: str) -> str: + return urlparse(url).path.rstrip("/").split("/")[-1] + + +def _versioned_release_path_segment(release_type: str, version_number: str) -> str: + if version_number == "NA": + return release_type + return f"{release_type}-{version_number}" + + +def _artifact_url_specificity(url: str, release_type: str, version_number: str) -> int: + """Prefer versioned firmware-server paths over generic release aliases.""" + segment = _versioned_release_path_segment(release_type, version_number) + if f"/{segment}/" in url: + return 2 + if f"/{release_type}/" in url: + return 1 + return 0 + + +def _dedupe_board_artifacts( + artifacts: list[BoardArtifact], + release_type: str, + version_number: str, +) -> list[BoardArtifact]: + """Manifest entries may alias the same file under versioned and generic URLs.""" + by_name: dict[str, BoardArtifact] = {} + for artifact in artifacts: + existing = by_name.get(artifact.name) + if existing is None or _artifact_url_specificity( + artifact.url, release_type, version_number + ) > _artifact_url_specificity(existing.url, release_type, version_number): + by_name[artifact.name] = artifact + return list(by_name.values()) + + def _release_fields_from_entry( entry: dict, -) -> Optional[tuple[str, str, str, str, str]]: +) -> Optional[tuple[str, str, str, str]]: vehicle_id = vehicle_id_for_manifest_entry(entry) if vehicle_id is None: return None @@ -106,7 +182,7 @@ def _release_fields_from_entry( return None version_number = "NA" if release_type == "latest" else manifest_version - return vehicle_id, release_type, version_number, base_url, git_sha + return vehicle_id, release_type, version_number, git_sha def _record_release_meta( @@ -114,7 +190,6 @@ def _record_release_meta( vehicle_id: str, release_type: str, version_number: str, - base_url: str, git_sha: str, logger: logging.Logger, ) -> None: @@ -124,7 +199,6 @@ def _record_release_meta( "vehicle_id": vehicle_id, "release_type": release_type, "version_number": version_number, - "ap_build_artifacts_url": base_url, "git_sha": git_sha, } return @@ -138,6 +212,30 @@ def _record_release_meta( ) +def _record_board_artifact( + artifacts_by_release: dict[tuple[str, str, str], dict[str, list[BoardArtifact]]], + entry: dict, + vehicle_id: str, + release_type: str, + version_number: str, +) -> None: + platform = entry.get("platform") + artifact_url = entry.get("url", "") + if not platform or not artifact_url: + return + + platform = manifest_platform_key(platform, vehicle_id) + key = (vehicle_id, release_type, version_number) + artifacts_by_release[key][platform].append( + BoardArtifact( + name=_artifact_name_from_url(artifact_url), + url=artifact_url, + format=entry.get("format", ""), + size=entry.get("image_size"), + ) + ) + + def _releases_from_meta( release_meta: dict[tuple, dict], ) -> dict[str, list[ReleaseRecord]]: @@ -149,7 +247,6 @@ def _releases_from_meta( release_type=meta["release_type"], version_number=meta["version_number"], commit_reference=meta["git_sha"], - ap_build_artifacts_url=meta["ap_build_artifacts_url"], ) ) @@ -168,34 +265,81 @@ def _releases_from_meta( class ManifestIndex: """In-memory index built from a parsed manifest document.""" - def __init__(self, releases_by_vehicle: dict[str, list[ReleaseRecord]]): + def __init__( + self, + releases_by_vehicle: dict[str, list[ReleaseRecord]], + artifacts_by_release: dict[ + tuple[str, str, str], dict[str, list[BoardArtifact]] + ], + ): self.releases_by_vehicle = releases_by_vehicle + self.artifacts_by_release = artifacts_by_release @classmethod def build(cls, manifest: dict) -> "ManifestIndex": logger = logging.getLogger(__name__) release_meta: dict[tuple, dict] = {} + artifacts_by_release: dict[ + tuple[str, str, str], dict[str, list[BoardArtifact]] + ] = defaultdict(lambda: defaultdict(list)) for entry in manifest.get("firmware") or []: fields = _release_fields_from_entry(entry) if fields is None: continue - vehicle_id, release_type, version_number, base_url, git_sha = fields + vehicle_id, release_type, version_number, git_sha = fields _record_release_meta( release_meta, vehicle_id, release_type, version_number, - base_url, git_sha, logger, ) + _record_board_artifact( + artifacts_by_release, + entry, + vehicle_id, + release_type, + version_number, + ) - return cls(_releases_from_meta(release_meta)) + return cls( + _releases_from_meta(release_meta), + { + release_key: dict(platforms) + for release_key, platforms in artifacts_by_release.items() + }, + ) def get_releases(self, vehicle_id: str) -> list[ReleaseRecord]: return list(self.releases_by_vehicle.get(vehicle_id, [])) + def get_board_artifacts( + self, + vehicle_id: str, + release_type: str, + version_number: str, + board_id: str, + ) -> list[BoardArtifact]: + key = (vehicle_id, release_type, version_number) + artifacts = list(self.artifacts_by_release.get(key, {}).get(board_id, [])) + return _dedupe_board_artifacts(artifacts, release_type, version_number) + + def get_features_txt_url( + self, + vehicle_id: str, + release_type: str, + version_number: str, + board_id: str, + ) -> Optional[str]: + artifacts = self.get_board_artifacts( + vehicle_id, release_type, version_number, board_id + ) + if not artifacts: + return None + return artifacts[0].url.rsplit("/", 1)[0] + "/features.txt" + def _version_sort_key(version_number: str) -> tuple: if version_number == "NA": diff --git a/metadata_manager/firmware_server/manifest.py b/metadata_manager/firmware_server/manifest.py index 6332f7f..3c48a1e 100644 --- a/metadata_manager/firmware_server/manifest.py +++ b/metadata_manager/firmware_server/manifest.py @@ -3,7 +3,8 @@ from .client import ManifestClient from .exceptions import ManifestFetchError -from .index import ManifestIndex +from .index import ManifestIndex, latest_features_txt_url +from .models import BoardArtifact class ManifestJSON: @@ -38,3 +39,37 @@ def get_releases(self, vehicle_id: str) -> list: if not self.is_available: return [] return self._index.get_releases(vehicle_id) + + def get_board_artifacts( + self, + vehicle_id: str, + release_type: str, + version_number: str, + board_id: str, + ) -> list[BoardArtifact]: + if not self.is_available: + return [] + return self._index.get_board_artifacts( + vehicle_id=vehicle_id, + release_type=release_type, + version_number=version_number, + board_id=board_id, + ) + + def get_features_txt_url( + self, + vehicle_id: str, + release_type: str, + version_number: str, + board_id: str, + ) -> Optional[str]: + if release_type == "tag": + return latest_features_txt_url(vehicle_id, board_id) + if not self.is_available: + return None + return self._index.get_features_txt_url( + vehicle_id=vehicle_id, + release_type=release_type, + version_number=version_number, + board_id=board_id, + ) diff --git a/metadata_manager/firmware_server/models.py b/metadata_manager/firmware_server/models.py index 3ce2455..f514a5c 100644 --- a/metadata_manager/firmware_server/models.py +++ b/metadata_manager/firmware_server/models.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Optional @dataclass(frozen=True) @@ -7,4 +8,11 @@ class ReleaseRecord: release_type: str version_number: str commit_reference: str - ap_build_artifacts_url: str + + +@dataclass(frozen=True) +class BoardArtifact: + name: str + url: str + format: str + size: Optional[int] = None diff --git a/metadata_manager/remotes.schema.json b/metadata_manager/remotes.schema.json index d6e676b..a207a62 100644 --- a/metadata_manager/remotes.schema.json +++ b/metadata_manager/remotes.schema.json @@ -39,10 +39,6 @@ "type": "string", "description": "Ardupilot version number for that release" }, - "ap_build_artifacts_url": { - "type": "string", - "description": "url to build artifacts at AP firmware server to fetch features.txt" - }, "commit_reference": { "type": "string", "description": "reference to commit for that release, this can be branch name, tag or git hash" diff --git a/metadata_manager/versions_manager/manager.py b/metadata_manager/versions_manager/manager.py index 7846199..6778f91 100644 --- a/metadata_manager/versions_manager/manager.py +++ b/metadata_manager/versions_manager/manager.py @@ -60,7 +60,6 @@ def __init__( ) self._providers = providers self.repo = ap_repo - self._remotes_json_path = remotes_json_path self.__task__runner = TaskRunner(tasks=((self.refresh_all, 1200),)) VersionsManager.__singleton = self diff --git a/metadata_manager/versions_manager/models.py b/metadata_manager/versions_manager/models.py index 3e9a780..bff1ae6 100644 --- a/metadata_manager/versions_manager/models.py +++ b/metadata_manager/versions_manager/models.py @@ -41,13 +41,11 @@ def __init__( commit_ref: str, release_type: str, version_number: str, - ap_build_artifacts_url, ) -> None: self.remote_info = remote_info self.commit_ref = commit_ref self.release_type = release_type self.version_number = version_number - self.ap_build_artifacts_url = ap_build_artifacts_url commit_ref_sanitized = commit_ref.replace("/", "-") commit_ref_hash = hashlib.md5(commit_ref.encode()).hexdigest()[:8] @@ -66,5 +64,4 @@ def from_release_record( commit_ref=release.commit_reference, release_type=release.release_type, version_number=release.version_number, - ap_build_artifacts_url=release.ap_build_artifacts_url, ) diff --git a/metadata_manager/versions_manager/providers.py b/metadata_manager/versions_manager/providers.py index 32336f2..aac2671 100644 --- a/metadata_manager/versions_manager/providers.py +++ b/metadata_manager/versions_manager/providers.py @@ -18,26 +18,6 @@ OFFICIAL_REMOTE_NAME = "ardupilot" OFFICIAL_REMOTE_URL = "https://github.com/ardupilot/ardupilot.git" -FIRMWARE_SERVER_BASE = "https://firmware.ardupilot.org" - -# CBS vehicle id -> firmware.ardupilot.org top-level directory. -_FIRMWARE_SERVER_DIR_BY_VEHICLE_ID = { - "copter": "Copter", - "plane": "Plane", - "rover": "Rover", - "sub": "Sub", - "heli": "Copter", - "blimp": "Blimp", - "tracker": "AntennaTracker", - "ap-periph": "AP_Periph", -} - - -def _firmware_server_dir(vehicle_id: str) -> str: - try: - return _FIRMWARE_SERVER_DIR_BY_VEHICLE_ID[vehicle_id] - except KeyError as exc: - raise ValueError(f"Unknown vehicle id: {vehicle_id}") from exc def _vehicle_id_for_tag_segment(segment: str) -> Optional[str]: @@ -187,10 +167,6 @@ def refresh(self) -> None: release_type="tag", version_number=parts[-1], commit_reference=tag_info["object"]["sha"], - ap_build_artifacts_url=( - f"{FIRMWARE_SERVER_BASE}/" - f"{_firmware_server_dir(vid)}/latest" - ), ) ) @@ -294,9 +270,6 @@ def get_versions(self, vehicle_id: str) -> list[VersionInfo]: commit_ref=release.get("commit_reference"), release_type=release.get("release_type"), version_number=release.get("version_number"), - ap_build_artifacts_url=release.get( - "ap_build_artifacts_url" - ), ) ) return versions diff --git a/tests/metadata_manager/fixtures/manifest_sample.json b/tests/metadata_manager/fixtures/manifest_sample.json index 3cf4a73..a692edc 100644 --- a/tests/metadata_manager/fixtures/manifest_sample.json +++ b/tests/metadata_manager/fixtures/manifest_sample.json @@ -13,6 +13,18 @@ "mav-type": "QUADROTOR", "image_size": 100 }, + { + "vehicletype": "Copter", + "platform": "CubeOrange", + "git-sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "url": "https://firmware.ardupilot.org/Copter/stable/CubeOrange/arducopter.apj", + "mav-firmware-version-type": "OFFICIAL", + "mav-firmware-version": "4.6.3", + "format": "apj", + "latest": 0, + "mav-type": "QUADROTOR", + "image_size": 100 + }, { "vehicletype": "Copter", "platform": "CubeOrange-heli", diff --git a/tests/metadata_manager/test_features_txt_client.py b/tests/metadata_manager/test_features_txt_client.py new file mode 100644 index 0000000..e8f52d0 --- /dev/null +++ b/tests/metadata_manager/test_features_txt_client.py @@ -0,0 +1,161 @@ +import requests +from unittest.mock import Mock, patch + +import dill + +from metadata_manager.firmware_server.features import ( + FeaturesTxtClient, + parse_features_txt, +) + + +class TestParseFeaturesTxt: + def test_parses_enabled_and_disabled_features(self): + text = "\n".join([ + "HAL_LOGGING_ENABLED", + "!HAL_PROFILER_ENABLED", + "# comment", + "", + "AP_FENCE_ENABLED", + ]) + assert parse_features_txt(text) == { + "HAL_LOGGING_ENABLED": 1, + "HAL_PROFILER_ENABLED": 0, + "AP_FENCE_ENABLED": 1, + } + + +class TestFeaturesTxtClient: + def test_get_defaults_fetches_and_caches_with_etag(self): + client = FeaturesTxtClient(caching_enabled=True) + client._redis_client = Mock() + client._redis_client.get.return_value = None + response = Mock( + status_code=200, + text="HAL_LOGGING_ENABLED\n!HAL_PROFILER_ENABLED", + headers={"ETag": '"abc"'}, + ) + response.raise_for_status = Mock() + + with patch( + "metadata_manager.firmware_server.features.requests.get", + return_value=response, + ) as mock_get: + result = client.get_defaults("https://example.com/features.txt") + + assert result == { + "HAL_LOGGING_ENABLED": 1, + "HAL_PROFILER_ENABLED": 0, + } + mock_get.assert_called_once() + client._redis_client.set.assert_called_once() + stored = dill.loads(client._redis_client.set.call_args.kwargs["value"]) + assert stored == { + "etag": '"abc"', + "defaults": result, + } + + def test_get_defaults_skips_cache_without_etag(self): + client = FeaturesTxtClient(caching_enabled=True) + client._redis_client = Mock() + client._redis_client.get.return_value = None + response = Mock(status_code=200, text="HAL_LOGGING_ENABLED", headers={}) + response.raise_for_status = Mock() + + with patch( + "metadata_manager.firmware_server.features.requests.get", + return_value=response, + ): + result = client.get_defaults("https://example.com/features.txt") + + assert result == {"HAL_LOGGING_ENABLED": 1} + client._redis_client.set.assert_not_called() + + def test_get_defaults_revalidates_cached_entry_with_if_none_match(self): + client = FeaturesTxtClient(caching_enabled=True) + client._redis_client = Mock() + cached = { + "etag": '"abc"', + "defaults": {"HAL_LOGGING_ENABLED": 1}, + } + client._redis_client.get.return_value = dill.dumps(cached) + response = Mock(status_code=304, headers={}) + response.raise_for_status = Mock() + + with patch( + "metadata_manager.firmware_server.features.requests.get", + return_value=response, + ) as mock_get: + result = client.get_defaults("https://example.com/features.txt") + + assert result == cached["defaults"] + mock_get.assert_called_once_with( + "https://example.com/features.txt", + headers={ + "User-Agent": "CustomBuild/1.0", + "Accept-Encoding": "identity", + "If-None-Match": '"abc"', + }, + timeout=30, + ) + client._redis_client.set.assert_not_called() + + def test_get_defaults_updates_cache_when_etag_changes(self): + client = FeaturesTxtClient(caching_enabled=True) + client._redis_client = Mock() + cached = { + "etag": '"old"', + "defaults": {"OLD_FEATURE": 1}, + } + client._redis_client.get.return_value = dill.dumps(cached) + response = Mock( + status_code=200, + text="NEW_FEATURE\n!OLD_FEATURE", + headers={"ETag": '"new"'}, + ) + response.raise_for_status = Mock() + + with patch( + "metadata_manager.firmware_server.features.requests.get", + return_value=response, + ) as mock_get: + result = client.get_defaults("https://example.com/features.txt") + + assert result == {"NEW_FEATURE": 1, "OLD_FEATURE": 0} + mock_get.assert_called_once_with( + "https://example.com/features.txt", + headers={ + "User-Agent": "CustomBuild/1.0", + "Accept-Encoding": "identity", + "If-None-Match": '"old"', + }, + timeout=30, + ) + stored = dill.loads(client._redis_client.set.call_args.kwargs["value"]) + assert stored["etag"] == '"new"' + assert stored["defaults"] == result + + def test_get_defaults_returns_none_on_fetch_failure(self): + client = FeaturesTxtClient(caching_enabled=False) + + with patch( + "metadata_manager.firmware_server.features.requests.get", + side_effect=requests.RequestException("network error"), + ): + result = client.get_defaults("https://example.com/features.txt") + + assert result is None + + def test_get_defaults_does_not_cache_failure_when_caching_enabled(self): + client = FeaturesTxtClient(caching_enabled=True) + client._redis_client = Mock() + client._redis_client.get.return_value = None + + with patch( + "metadata_manager.firmware_server.features.requests.get", + side_effect=requests.RequestException("network error"), + ): + result = client.get_defaults("https://example.com/missing/features.txt") + + assert result is None + client._redis_client.set.assert_not_called() diff --git a/tests/metadata_manager/test_firmware_server.py b/tests/metadata_manager/test_firmware_server.py index 518720a..55c8e71 100644 --- a/tests/metadata_manager/test_firmware_server.py +++ b/tests/metadata_manager/test_firmware_server.py @@ -2,8 +2,9 @@ from pathlib import Path from unittest.mock import Mock, patch -from metadata_manager import ManifestClient, ManifestIndex +from metadata_manager import ManifestClient, ManifestIndex, ManifestJSON from metadata_manager.firmware_server.client import _CacheMeta +from metadata_manager.firmware_server.index import latest_features_txt_url FIXTURES_DIR = Path(__file__).parent / "fixtures" @@ -31,10 +32,115 @@ def test_builds_releases_for_copter_heli_and_tracker(self): assert len(latest) == 1 assert latest[0].commit_reference.startswith("eeee") assert latest[0].version_number == "NA" - assert latest[0].ap_build_artifacts_url == ( - "https://firmware.ardupilot.org/Copter/latest" + + def test_indexes_board_artifacts_for_vehicle_version_board(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + copter_artifacts = index.get_board_artifacts( + "copter", "stable", "4.6.3", "CubeOrange" + ) + assert len(copter_artifacts) == 1 + assert copter_artifacts[0].name == "arducopter.apj" + assert copter_artifacts[0].format == "apj" + assert copter_artifacts[0].url.endswith("/Copter/stable-4.6.3/CubeOrange/arducopter.apj") + + heli_artifacts = index.get_board_artifacts( + "heli", "stable", "4.6.3", "CubeOrange" + ) + assert len(heli_artifacts) == 1 + assert heli_artifacts[0].name == "arducopter-heli.apj" + + latest_artifacts = index.get_board_artifacts( + "copter", "latest", "NA", "CubeOrange" + ) + assert len(latest_artifacts) == 1 + assert latest_artifacts[0].url.endswith("/Copter/latest/CubeOrange/arducopter.apj") + + def test_board_artifacts_dedupe_generic_stable_alias(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + copter_artifacts = index.get_board_artifacts( + "copter", "stable", "4.6.3", "CubeOrange" + ) + assert len(copter_artifacts) == 1 + assert copter_artifacts[0].url.endswith("/Copter/stable-4.6.3/CubeOrange/arducopter.apj") + + heli_artifacts = index.get_board_artifacts( + "heli", "stable", "4.6.3", "CubeOrange" + ) + assert len(heli_artifacts) == 1 + assert heli_artifacts[0].name == "arducopter-heli.apj" + + latest_artifacts = index.get_board_artifacts( + "copter", "latest", "NA", "CubeOrange" + ) + assert len(latest_artifacts) == 1 + assert latest_artifacts[0].url.endswith("/Copter/latest/CubeOrange/arducopter.apj") + + def test_board_artifacts_missing_returns_empty_list(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + assert index.get_board_artifacts("copter", "stable", "4.6.3", "UnknownBoard") == [] + + def test_get_features_txt_url_from_manifest_artifact(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + url = index.get_features_txt_url("copter", "stable", "4.6.3", "CubeOrange") + assert url == ( + "https://firmware.ardupilot.org/Copter/stable-4.6.3/" + "CubeOrange/features.txt" + ) + + heli_url = index.get_features_txt_url("heli", "stable", "4.6.3", "CubeOrange") + assert heli_url == ( + "https://firmware.ardupilot.org/Copter/stable-4.6.3/" + "CubeOrange-heli/features.txt" ) + def test_get_features_txt_url_missing_board_returns_none(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + assert index.get_features_txt_url( + "copter", "stable", "4.6.3", "UnknownBoard" + ) is None + + +class TestLatestFeaturesTxtUrl: + def test_copter_board(self): + assert latest_features_txt_url("copter", "CubeOrange") == ( + "https://firmware.ardupilot.org/Copter/latest/CubeOrange/features.txt" + ) + + def test_heli_board(self): + assert latest_features_txt_url("heli", "CubeOrange") == ( + "https://firmware.ardupilot.org/Copter/latest/CubeOrange-heli/features.txt" + ) + + +class TestManifestJSONFeaturesUrl: + def test_tag_release_uses_latest_url(self): + manifest_json = ManifestJSON(url="https://example.com/manifest.json", cache_dir="/tmp") + assert manifest_json.get_features_txt_url( + "copter", "tag", "my-feature", "CubeOrange" + ) == latest_features_txt_url("copter", "CubeOrange") + + def test_stable_release_uses_manifest_index(self): + manifest_json = ManifestJSON(url="https://example.com/manifest.json", cache_dir="/tmp") + manifest_json._index = ManifestIndex.build(SAMPLE_MANIFEST) + + assert manifest_json.get_features_txt_url( + "copter", "stable", "4.6.3", "CubeOrange" + ) == ( + "https://firmware.ardupilot.org/Copter/stable-4.6.3/" + "CubeOrange/features.txt" + ) + + def test_unavailable_manifest_returns_none_for_stable(self): + manifest_json = ManifestJSON(url="https://example.com/manifest.json", cache_dir="/tmp") + assert manifest_json.get_features_txt_url( + "copter", "stable", "4.6.3", "CubeOrange" + ) is None + class TestManifestClientCache: def test_uses_cache_on_304(self, tmp_path): diff --git a/tests/metadata_manager/test_versions_manager.py b/tests/metadata_manager/test_versions_manager.py index ab6915f..d80576e 100644 --- a/tests/metadata_manager/test_versions_manager.py +++ b/tests/metadata_manager/test_versions_manager.py @@ -71,7 +71,6 @@ def _make_version(self, release_type: str, version_number: str, commit_ref: str) commit_ref=commit_ref, release_type=release_type, version_number=version_number, - ap_build_artifacts_url=None, ) def test_stable_wins_when_beta_is_seen_first(self, versions_manager): diff --git a/tests/web/conftest.py b/tests/web/conftest.py index 51b931d..302bdc8 100644 --- a/tests/web/conftest.py +++ b/tests/web/conftest.py @@ -247,6 +247,8 @@ async def test_lifespan(app: FastAPI): # Create mock AP source metadata fetcher mock_ap_src_fetcher = Mock() app.state.ap_src_metadata_fetcher = mock_ap_src_fetcher + app.state.manifest_json = Mock() + app.state.features_txt_client = Mock() # Don't start background tasks in test mode # versions_manager.start() diff --git a/tests/web/test_builds_service.py b/tests/web/test_builds_service.py index 03b2453..8270ec2 100644 --- a/tests/web/test_builds_service.py +++ b/tests/web/test_builds_service.py @@ -53,14 +53,12 @@ def make_version_info( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ): return VersionInfo( remote_info=RemoteInfo(name=remote_name, url=remote_url), commit_ref=commit_ref, release_type=release_type, version_number=version_number, - ap_build_artifacts_url=ap_build_artifacts_url, ) diff --git a/tests/web/test_vehicles_api.py b/tests/web/test_vehicles_api.py index c935654..74b866b 100644 --- a/tests/web/test_vehicles_api.py +++ b/tests/web/test_vehicles_api.py @@ -9,6 +9,7 @@ VehicleBase, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, CategoryBase, FeatureDefault, @@ -59,6 +60,15 @@ def dummy_board( version_id=version_id, ) + @staticmethod + def dummy_standard_artifact(): + return StandardArtifactOut( + name="arducopter.apj", + url="https://firmware.ardupilot.org/Copter/stable-4.6.3/CubeOrange/arducopter.apj", + format="apj", + size=100, + ) + @staticmethod def dummy_feature( vehicle_id="copter", @@ -442,6 +452,56 @@ def test_get_board_method_not_allowed(self, client): response = method("/api/v1/vehicles/copter/versions/v1/boards/b1") assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + # GET /vehicles/{vehicle_id}/versions/{version_id}/boards/{board_id}/standard_artifacts + + _STANDARD_ARTIFACTS_URL = ( + "/api/v1/vehicles/copter/versions/copter-4.5.0-stable/" + "boards/MatekH743/standard_artifacts" + ) + + def test_list_board_standard_artifacts_returns_200(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + assert response.status_code == status.HTTP_200_OK + + def test_list_board_standard_artifacts_returns_404_when_not_found(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = None + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_list_board_standard_artifacts_response_schema(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + data = response.json() + assert data[0]["name"] == "arducopter.apj" + assert data[0]["format"] == "apj" + assert "url" in data[0] + + def test_list_board_standard_artifacts_service_called_with_correct_ids(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + client.get(self._STANDARD_ARTIFACTS_URL) + + mock_vehicles_service.get_board_standard_artifacts.assert_called_once_with( + "copter", "copter-4.5.0-stable", "MatekH743" + ) + # GET /vehicles/{vehicle_id}/versions/{version_id}/boards/{board_id}/features _FEATURES_URL = "/api/v1/vehicles/copter/versions/copter-4.5.0-stable/boards/MatekH743/features" diff --git a/tests/web/test_vehicles_service.py b/tests/web/test_vehicles_service.py index bb8f31c..cb42d6b 100644 --- a/tests/web/test_vehicles_service.py +++ b/tests/web/test_vehicles_service.py @@ -5,6 +5,7 @@ from unittest.mock import Mock from metadata_manager import ( + BoardArtifact, RemoteInfo, Vehicle, VersionInfo, @@ -13,11 +14,34 @@ @pytest.fixture -def service(mock_vehicles_manager, mock_versions_manager, mock_ap_src_metadata_fetcher, mock_git_repo): +def mock_manifest_json(): + mock = Mock() + mock.get_features_txt_url.return_value = None + return mock + + +@pytest.fixture +def mock_features_txt_client(): + mock = Mock() + mock.get_defaults.return_value = None + return mock + + +@pytest.fixture +def service( + mock_vehicles_manager, + mock_versions_manager, + mock_ap_src_metadata_fetcher, + mock_manifest_json, + mock_features_txt_client, + mock_git_repo, +): return VehiclesService( vehicle_manager=mock_vehicles_manager, versions_manager=mock_versions_manager, ap_src_metadata_fetcher=mock_ap_src_metadata_fetcher, + manifest_json=mock_manifest_json, + features_txt_client=mock_features_txt_client, repo=mock_git_repo, ) @@ -154,7 +178,6 @@ def test_get_versions_single(self, service, mock_versions_manager): commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -169,21 +192,18 @@ def test_get_versions_many(self, service, mock_versions_manager): commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -198,21 +218,18 @@ def test_get_versions_sorted_by_name(self, service, mock_versions_manager): commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -241,21 +258,18 @@ def test_get_versions_type_filter_keeps_matching( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter", type_filter="stable") @@ -273,14 +287,12 @@ def test_get_versions_type_filter_excludes_non_matching( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter", type_filter="latest") @@ -297,21 +309,18 @@ def test_get_versions_type_filter_none_returns_all( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -328,21 +337,18 @@ def test_get_versions_type_filter_multiple_matches( commit_ref="refs/tags/Copter-4.4.0", release_type="stable", version_number="4.4.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter", type_filter="stable") @@ -360,7 +366,6 @@ def test_get_versions_latest_name_format( commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -377,7 +382,6 @@ def test_get_versions_non_latest_name_format( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), ] versions = service.get_versions("copter") @@ -393,7 +397,6 @@ def test_get_version_found(self, service, mock_versions_manager): commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_versions_for_vehicle.return_value = [version_info] @@ -410,7 +413,6 @@ def test_get_version_not_found(self, service, mock_versions_manager): commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ), ] @@ -433,21 +435,18 @@ def test_get_version_returns_correct_match_among_many(self, service, mock_versio commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) beta_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.6.0-beta", release_type="beta", version_number="4.6.0", - ap_build_artifacts_url=None, ) latest_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", release_type="latest", version_number="NA", - ap_build_artifacts_url=None, ) mock_versions_manager.get_versions_for_vehicle.return_value = [ stable_info, beta_info, latest_info, @@ -491,7 +490,6 @@ def test_get_boards_empty( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [] @@ -509,7 +507,6 @@ def test_get_boards_single( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] @@ -529,7 +526,6 @@ def test_get_boards_many( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [ @@ -550,7 +546,6 @@ def test_get_boards_sets_correct_vehicle_and_version_ids( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] @@ -569,7 +564,6 @@ def test_get_boards_fetcher_called_with_correct_params( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [] @@ -591,7 +585,6 @@ def test_get_board_found(self, service, mock_versions_manager, mock_ap_src_metad commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed", "CubeOrange"] @@ -611,7 +604,6 @@ def test_get_board_not_found(self, service, mock_versions_manager, mock_ap_src_m commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed", "CubeOrange"] @@ -629,7 +621,6 @@ def test_get_board_returns_correct_match_among_many( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [ @@ -662,7 +653,6 @@ def test_get_features_zero_options_returns_empty( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [] @@ -680,7 +670,6 @@ def test_get_features_one_option( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label = "HAL_LOGGING_ENABLED" @@ -691,7 +680,6 @@ def test_get_features_one_option( opt.dependency = None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -708,7 +696,6 @@ def test_get_features_many_options( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_logging = Mock() opt_logging.label, opt_logging.define, opt_logging.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" @@ -721,7 +708,6 @@ def test_get_features_many_options( opt_sensors.description, opt_sensors.default, opt_sensors.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf, opt_sensors] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -736,7 +722,6 @@ def test_get_features_sorted_by_category( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_z = Mock() opt_z.label, opt_z.define, opt_z.category = "FEATURE_Z", "DEFINE_Z", "Sensors" @@ -749,22 +734,21 @@ def test_get_features_sorted_by_category( opt_m.description, opt_m.default, opt_m.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_z, opt_a, opt_m] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") assert [f.category.name for f in result] == ["EKF", "Logging", "Sensors"] - def test_get_features_uses_fallback_defaults_when_no_artifacts_url( - self, service, mock_versions_manager, mock_ap_src_metadata_fetcher + def test_get_features_uses_fallback_defaults_when_no_features_url( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, ): - """Test that build-options-py defaults are used when ap_build_artifacts_url is None.""" + """Test that build-options-py defaults are used when no features.txt URL is available.""" version_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_on = Mock() opt_on.label, opt_on.define, opt_on.category = "FEATURE_ON", "DEFINE_ON", "Cat" @@ -774,6 +758,7 @@ def test_get_features_uses_fallback_defaults_when_no_artifacts_url( opt_off.description, opt_off.default, opt_off.dependency = "", 0, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_on, opt_off] + mock_manifest_json.get_features_txt_url.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -782,10 +767,11 @@ def test_get_features_uses_fallback_defaults_when_no_artifacts_url( assert by_id["FEATURE_ON"].source == "build-options-py" assert by_id["FEATURE_OFF"].enabled is False assert by_id["FEATURE_OFF"].source == "build-options-py" - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.assert_not_called() + mock_features_txt_client.get_defaults.assert_not_called() def test_get_features_uses_firmware_server_defaults_when_available( - self, service, mock_versions_manager, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, ): """Test that firmware-server defaults override build-options-py when present.""" version_info = VersionInfo( @@ -793,7 +779,6 @@ def test_get_features_uses_firmware_server_defaults_when_available( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", ) opt_a = Mock() opt_a.label, opt_a.define, opt_a.category = "FEATURE_A", "DEFINE_A", "Cat" @@ -803,8 +788,11 @@ def test_get_features_uses_firmware_server_defaults_when_available( opt_b.description, opt_b.default, opt_b.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b] - # firmware server says DEFINE_A is disabled, DEFINE_B is enabled - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = { + features_url = ( + "https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeRed/features.txt" + ) + mock_manifest_json.get_features_txt_url.return_value = features_url + mock_features_txt_client.get_defaults.return_value = { "DEFINE_A": 0, "DEFINE_B": 1, } @@ -816,9 +804,11 @@ def test_get_features_uses_firmware_server_defaults_when_available( assert by_id["FEATURE_A"].source == "firmware-server" assert by_id["FEATURE_B"].enabled is True assert by_id["FEATURE_B"].source == "firmware-server" + mock_features_txt_client.get_defaults.assert_called_once_with(features_url) def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( - self, service, mock_versions_manager, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, ): """Test that build-options-py fallback is used when firmware server fetch fails.""" version_info = VersionInfo( @@ -826,14 +816,16 @@ def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None + mock_manifest_json.get_features_txt_url.return_value = ( + "https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeRed/features.txt" + ) + mock_features_txt_client.get_defaults.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -841,7 +833,8 @@ def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( assert result[0].default.source == "build-options-py" def test_get_features_firmware_server_overrides_only_known_defines( - self, service, mock_versions_manager, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, ): """Test that a define absent from firmware-server data falls back to build-options-py.""" version_info = VersionInfo( @@ -849,7 +842,6 @@ def test_get_features_firmware_server_overrides_only_known_defines( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", ) opt_known = Mock() opt_known.label, opt_known.define, opt_known.category = "FEATURE_KNOWN", "DEFINE_KNOWN", "Cat" @@ -859,8 +851,10 @@ def test_get_features_firmware_server_overrides_only_known_defines( opt_unknown.description, opt_unknown.default, opt_unknown.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_known, opt_unknown] - # firmware server only knows about DEFINE_KNOWN - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = { + mock_manifest_json.get_features_txt_url.return_value = ( + "https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeRed/features.txt" + ) + mock_features_txt_client.get_defaults.return_value = { "DEFINE_KNOWN": 1, } @@ -872,6 +866,65 @@ def test_get_features_firmware_server_overrides_only_known_defines( assert by_id["FEATURE_UNKNOWN"].enabled is True assert by_id["FEATURE_UNKNOWN"].source == "build-options-py" + def test_get_features_tag_version_uses_latest_url( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, + ): + """Fork tag builds use hardcoded latest features.txt URL.""" + version_info = VersionInfo( + remote_info=RemoteInfo(name="shiv-tyagi", url="https://github.com/shiv-tyagi/ardupilot.git"), + commit_ref="refs/tags/custom-build/my-feature", + release_type="tag", + version_number="my-feature", + ) + opt = Mock() + opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" + opt.description, opt.default, opt.dependency = "", 0, None + mock_versions_manager.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] + latest_url = ( + "https://firmware.ardupilot.org/Copter/latest/CubeOrange/features.txt" + ) + mock_features_txt_client.get_defaults.return_value = {"DEFINE_A": 1} + + result = service.get_features("copter", version_info.version_id, "CubeOrange") + + mock_manifest_json.get_features_txt_url.assert_not_called() + mock_features_txt_client.get_defaults.assert_called_once_with(latest_url) + assert result[0].default.enabled is True + assert result[0].default.source == "firmware-server" + + def test_get_features_fork_remote_uses_latest_url( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, mock_features_txt_client, + ): + """Non-official fork remotes use latest ardupilot features.txt for any release.""" + version_info = VersionInfo( + remote_info=RemoteInfo( + name="test-remote-1", + url="https://github.com/test/ardupilot.git", + ), + commit_ref="refs/heads/master", + release_type="latest", + version_number="4.6.0", + ) + opt = Mock() + opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" + opt.description, opt.default, opt.dependency = "", 1, None + mock_versions_manager.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] + latest_url = ( + "https://firmware.ardupilot.org/Copter/latest/CubeOrange/features.txt" + ) + mock_features_txt_client.get_defaults.return_value = {"DEFINE_A": 0} + + result = service.get_features("copter", version_info.version_id, "CubeOrange") + + mock_manifest_json.get_features_txt_url.assert_not_called() + mock_features_txt_client.get_defaults.assert_called_once_with(latest_url) + assert result[0].default.enabled is False + assert result[0].default.source == "firmware-server" + def test_get_features_dependency_none( self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): @@ -881,14 +934,12 @@ def test_get_features_dependency_none( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -903,14 +954,12 @@ def test_get_features_dependency_single( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE" mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -925,14 +974,12 @@ def test_get_features_dependency_multiple_comma_separated( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE,DEP_TWO,DEP_THREE" mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -947,14 +994,12 @@ def test_get_features_dependency_with_spaces( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE , DEP_TWO , DEP_THREE" mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -969,14 +1014,12 @@ def test_get_features_ids_filled_correctly( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -993,7 +1036,6 @@ def test_get_features_category_filter_keeps_matching( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_logging = Mock() opt_logging.label, opt_logging.define, opt_logging.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" @@ -1006,7 +1048,6 @@ def test_get_features_category_filter_keeps_matching( opt_sensors.description, opt_sensors.default, opt_sensors.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf, opt_sensors] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed", category_id="Logging") @@ -1023,7 +1064,6 @@ def test_get_features_category_filter_excludes_non_matching( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_logging = Mock() opt_logging.label, opt_logging.define, opt_logging.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" @@ -1033,7 +1073,6 @@ def test_get_features_category_filter_excludes_non_matching( opt_ekf.description, opt_ekf.default, opt_ekf.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed", category_id="Sensors") @@ -1048,7 +1087,6 @@ def test_get_features_category_filter_no_matches_returns_empty( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_a = Mock() opt_a.label, opt_a.define, opt_a.category = "FEATURE_A", "DEFINE_A", "Logging" @@ -1058,7 +1096,6 @@ def test_get_features_category_filter_no_matches_returns_empty( opt_b.description, opt_b.default, opt_b.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_features("copter", version_info.version_id, "CubeRed", category_id="NonExistent") @@ -1075,14 +1112,12 @@ def test_get_feature_found( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" opt.description, opt.default, opt.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_feature("copter", version_info.version_id, "CubeRed", "HAL_LOGGING_ENABLED") @@ -1099,14 +1134,12 @@ def test_get_feature_not_found( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt = Mock() opt.label, opt.define, opt.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" opt.description, opt.default, opt.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_feature("copter", version_info.version_id, "CubeRed", "NONEXISTENT_FEATURE") @@ -1121,7 +1154,6 @@ def test_get_feature_returns_correct_match_among_many( commit_ref="refs/tags/Copter-4.5.0", release_type="stable", version_number="4.5.0", - ap_build_artifacts_url=None, ) opt_a = Mock() opt_a.label, opt_a.define, opt_a.category = "FEATURE_A", "DEFINE_A", "Cat" @@ -1134,10 +1166,107 @@ def test_get_feature_returns_correct_match_among_many( opt_c.description, opt_c.default, opt_c.dependency = "", 1, None mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b, opt_c] - mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None result = service.get_feature("copter", version_info.version_id, "CubeRed", "FEATURE_B") assert result is not None assert result.id == "FEATURE_B" assert result.default.enabled is False + + +class TestGetBoardStandardArtifacts: + def test_board_not_found_returns_none( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, + ): + remote_info = Mock() + remote_info.name = "ardupilot" + mock_versions_manager.get_version_info.return_value = Mock( + version_id="v1", + commit_ref="abc123", + remote_info=remote_info, + ) + mock_ap_src_metadata_fetcher.get_boards.return_value = [] + + result = service.get_board_standard_artifacts("copter", "v1", "CubeOrange") + + assert result is None + mock_manifest_json.get_board_artifacts.assert_not_called() + + def test_success( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, + ): + remote_info = Mock() + remote_info.name = "ardupilot" + version_info = Mock( + version_id="v1", + commit_ref="abc123", + release_type="stable", + version_number="4.6.3", + remote_info=remote_info, + ) + mock_versions_manager.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeOrange"] + mock_manifest_json.get_board_artifacts.return_value = [ + BoardArtifact( + name="arducopter.apj", + url="https://firmware.ardupilot.org/Copter/stable-4.6.3/CubeOrange/arducopter.apj", + format="apj", + size=100, + ) + ] + + result = service.get_board_standard_artifacts("copter", "v1", "CubeOrange") + + assert len(result) == 1 + assert result[0].name == "arducopter.apj" + assert result[0].format == "apj" + mock_manifest_json.get_board_artifacts.assert_called_once_with( + vehicle_id="copter", + release_type="stable", + version_number="4.6.3", + board_id="CubeOrange", + ) + + def test_not_in_manifest_returns_none( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, + ): + remote_info = Mock() + remote_info.name = "ardupilot" + version_info = Mock( + version_id="v1", + commit_ref="abc123", + release_type="stable", + version_number="4.6.3", + remote_info=remote_info, + ) + mock_versions_manager.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeOrange"] + mock_manifest_json.get_board_artifacts.return_value = [] + + result = service.get_board_standard_artifacts("copter", "v1", "CubeOrange") + + assert result is None + + def test_fork_remote_returns_none_without_manifest_lookup( + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher, + mock_manifest_json, + ): + remote_info = Mock() + remote_info.name = "shiv-tyagi" + version_info = Mock( + version_id="v1", + commit_ref="abc123", + release_type="tag", + version_number="my-feature", + remote_info=remote_info, + ) + mock_versions_manager.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeOrange"] + + result = service.get_board_standard_artifacts("copter", "v1", "CubeOrange") + + assert result is None + mock_manifest_json.get_board_artifacts.assert_not_called() diff --git a/web/api/v1/vehicles.py b/web/api/v1/vehicles.py index 0fe97ee..01380bf 100644 --- a/web/api/v1/vehicles.py +++ b/web/api/v1/vehicles.py @@ -5,6 +5,7 @@ VehicleBase, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, ) from web.services.vehicles import get_vehicles_service, VehiclesService @@ -183,6 +184,38 @@ async def get_board( return board +@router.get( + "/{vehicle_id}/versions/{version_id}/boards/{board_id}/standard_artifacts", + response_model=List[StandardArtifactOut], + responses={ + 404: {"description": "Standard artifacts not found"}, + }, +) +async def list_board_standard_artifacts( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + board_id: str = Path(..., description="Board identifier"), + service: VehiclesService = Depends(get_vehicles_service), +): + """ + Get standard build artifact files from firmware.ardupilot.org for a board. + + Only official manifest-backed versions include per-board artifact URLs. + """ + artifacts = service.get_board_standard_artifacts( + vehicle_id, version_id, board_id + ) + if artifacts is None: + raise HTTPException( + status_code=404, + detail=( + f"Standard artifacts not found for board '{board_id}' " + f"in vehicle '{vehicle_id}' version '{version_id}'" + ), + ) + return artifacts + + # --- Feature Endpoints --- @router.get( "/{vehicle_id}/versions/{version_id}/boards/{board_id}/features", diff --git a/web/main.py b/web/main.py index 0925061..827a1d7 100755 --- a/web/main.py +++ b/web/main.py @@ -26,6 +26,7 @@ import build_manager from metadata_manager import ( APSourceMetadataFetcher, + FeaturesTxtClient, ManifestJSON, VehiclesManager, VersionsManager, @@ -64,6 +65,12 @@ async def lifespan(app: FastAPI): redis_port=settings.redis_port, ) + features_txt_client = FeaturesTxtClient( + redis_host=settings.redis_host, + redis_port=settings.redis_port, + caching_enabled=True, + ) + versions_manager = VersionsManager( ap_repo=repo, remotes_json_path=settings.remotes_json_path, @@ -101,6 +108,7 @@ async def lifespan(app: FastAPI): app.state.repo = repo app.state.ap_src_metadata_fetcher = ap_src_metadata_fetcher app.state.manifest_json = manifest_json + app.state.features_txt_client = features_txt_client app.state.versions_manager = versions_manager app.state.vehicles_manager = vehicles_manager app.state.build_manager = build_mgr diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py index dbeeaa5..7d31acf 100644 --- a/web/schemas/__init__.py +++ b/web/schemas/__init__.py @@ -27,6 +27,7 @@ VersionOut, BoardBase, BoardOut, + StandardArtifactOut, CategoryBase, FeatureDefault, FeatureBase, @@ -49,6 +50,7 @@ "VersionOut", "BoardBase", "BoardOut", + "StandardArtifactOut", "CategoryBase", "FeatureDefault", "FeatureBase", diff --git a/web/schemas/vehicles.py b/web/schemas/vehicles.py index 64ac43c..825562b 100644 --- a/web/schemas/vehicles.py +++ b/web/schemas/vehicles.py @@ -49,6 +49,13 @@ class BoardOut(BoardBase): version_id: str = Field(..., description="Associated version identifier") +class StandardArtifactOut(BaseModel): + name: str = Field(..., description="Artifact filename") + url: str = Field(..., description="Download URL on firmware.ardupilot.org") + format: str = Field(..., description="Artifact format (e.g. apj)") + size: Optional[int] = Field(None, description="File size in bytes") + + # --- Features --- class CategoryBase(BaseModel): id: str = Field(..., description="Unique category identifier") diff --git a/web/services/vehicles.py b/web/services/vehicles.py index ecf4d14..ebde485 100644 --- a/web/services/vehicles.py +++ b/web/services/vehicles.py @@ -5,11 +5,14 @@ from typing import List, Optional from fastapi import Request +from metadata_manager.firmware_server.index import latest_features_txt_url +from metadata_manager.versions_manager.providers import OFFICIAL_REMOTE_NAME from web.schemas import ( VehicleBase, RemoteInfo, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, CategoryBase, FeatureDefault, @@ -25,10 +28,14 @@ class VehiclesService: def __init__(self, vehicle_manager=None, versions_manager=None, ap_src_metadata_fetcher=None, + manifest_json=None, + features_txt_client=None, repo=None): self.vehicles_manager = vehicle_manager self.versions_manager = versions_manager self.ap_src_metadata_fetcher = ap_src_metadata_fetcher + self.manifest_json = manifest_json + self.features_txt_client = features_txt_client self.repo = repo def get_all_vehicles(self) -> List[VehicleBase]: @@ -150,6 +157,58 @@ def get_board( return board return None + def get_board_standard_artifacts( + self, + vehicle_id: str, + version_id: str, + board_id: str, + ) -> Optional[List[StandardArtifactOut]]: + """Get standard build artifacts for an official manifest-backed version.""" + version_info = self.versions_manager.get_version_info( + vehicle_id=vehicle_id, + version_id=version_id, + ) + if ( + version_info is None + or version_info.remote_info.name != OFFICIAL_REMOTE_NAME + ): + return None + + with self.repo.get_checkout_lock(): + boards = self.ap_src_metadata_fetcher.get_boards( + remote=version_info.remote_info.name, + commit_ref=version_info.commit_ref, + vehicle_id=vehicle_id, + ) + if board_id not in boards: + return None + + logger.info( + "Standard artifacts requested for %s version %s board %s", + vehicle_id, + version_id, + board_id, + ) + + artifacts = self.manifest_json.get_board_artifacts( + vehicle_id=vehicle_id, + release_type=version_info.release_type, + version_number=version_info.version_number, + board_id=board_id, + ) + if not artifacts: + return None + + return [ + StandardArtifactOut( + name=artifact.name, + url=artifact.url, + format=artifact.format, + size=artifact.size, + ) + for artifact in artifacts + ] + def get_features( self, vehicle_id: str, @@ -182,16 +241,20 @@ def get_features( ) # Try to fetch board-specific defaults from firmware-server - board_defaults = None - artifacts_dir = version_info.ap_build_artifacts_url - if artifacts_dir is not None: - board_defaults = ( - self.ap_src_metadata_fetcher.get_board_defaults_from_fw_server( - artifacts_url=artifacts_dir, - board_id=board_id, - vehicle_id=vehicle_id, - ) + if version_info.remote_info.name == OFFICIAL_REMOTE_NAME: + features_url = self.manifest_json.get_features_txt_url( + vehicle_id=vehicle_id, + release_type=version_info.release_type, + version_number=version_info.version_number, + board_id=board_id, ) + else: + features_url = latest_features_txt_url(vehicle_id, board_id) + board_defaults = ( + self.features_txt_client.get_defaults(features_url) + if features_url + else None + ) # Build feature list features = [] @@ -269,5 +332,7 @@ def get_vehicles_service(request: Request) -> VehiclesService: vehicle_manager=request.app.state.vehicles_manager, versions_manager=request.app.state.versions_manager, ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher, + manifest_json=request.app.state.manifest_json, + features_txt_client=request.app.state.features_txt_client, repo=request.app.state.repo, )