Add standard artifacts API endpoint and manifest.json backed feature defaults path

examples/remotes.json.sample

@@ -9,7 +9,6 @@
           {
             "release_type": "latest",
             "version_number": "4.6.0",
-            "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/latest",
             "commit_reference": "refs/heads/master"
           }
         ]
@@ -42,13 +41,11 @@
           {
             "release_type": "latest",
             "version_number": "4.6.0",
-            "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/latest",
             "commit_reference": "202cc6ae9d326a172be5ec1120d79595a6ddae36"
           },
           {
             "release_type": "stable",
             "version_number": "4.3.0",
-            "ap_build_artifacts_url": "https://firmware.ardupilot.org/Copter/stable-4.3.0",
             "commit_reference": "93448b71380c417644c9082b7b23e80fb982b626"
           }
         ]

metadata_manager/__init__.py

@@ -1,5 +1,5 @@
 from .ap_src_meta_fetcher import APSourceMetadataFetcher
-from .firmware_server import ManifestJSON, ManifestFetchError, ReleaseRecord
+from .firmware_server import BoardArtifact, FeaturesTxtClient, ManifestJSON, ManifestFetchError, ReleaseRecord
 from .firmware_server.client import ManifestClient
 from .firmware_server.index import ManifestIndex
 from .vehicles_manager import DEFAULT_VEHICLES, Vehicle, VehiclesManager
@@ -18,6 +18,8 @@
 
 __all__ = [
     "APSourceMetadataFetcher",
+    "BoardArtifact",
+    "FeaturesTxtClient",
     "DEFAULT_VEHICLES",
     "DEFAULT_WHITELISTED_FORK_REMOTES",
     "ForkRemoteSpec",

metadata_manager/ap_src_meta_fetcher.py

@@ -478,79 +478,6 @@ def get_build_options_at_commit(self, remote: str,
         )
         return build_options
 
-    def get_board_defaults_from_fw_server(
-        self,
-        artifacts_url: str,
-        board_id: str,
-        vehicle_id: str = None,
-    ) -> dict:
-        """
-        Fetch board defaults from firmware.ardupilot.org features.txt.
-
-        The features.txt file contains lines like:
-        - FEATURE_NAME (enabled features)
-        - !FEATURE_NAME (disabled features)
-
-        Parameters:
-            artifacts_url (str): Base URL for build artifacts for a version.
-            board_id (str): Board identifier
-            vehicle_id (str): Vehicle identifier
-                              (for special handling like Heli)
-
-        Returns:
-            dict: Dictionary mapping feature define to state
-                  (1 for enabled, 0 for disabled), or None if fetch fails
-        """
-        import requests
-
-        # Heli builds are stored under a separate folder
-        artifacts_subdir = board_id
-        if vehicle_id == "heli":
-            artifacts_subdir += "-heli"
-
-        features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt"
-
-        try:
-            response = requests.get(features_txt_url, timeout=30)
-            response.raise_for_status()
-
-            feature_states = {}
-            enabled_count = 0
-            disabled_count = 0
-
-            for line in response.text.splitlines():
-                line = line.strip()
-
-                # Skip empty lines and comments
-                if not line or line.startswith('#'):
-                    continue
-
-                # Check if feature is disabled (prefixed with !)
-                if line.startswith('!'):
-                    feature_name = line[1:].strip()
-                    if feature_name:
-                        feature_states[feature_name] = 0
-                        disabled_count += 1
-                else:
-                    # Enabled feature
-                    if line:
-                        feature_states[line] = 1
-                        enabled_count += 1
-
-            self.logger.info(
-                f"Fetched board defaults from firmware server: "
-                f"{enabled_count} enabled, "
-                f"{disabled_count} disabled"
-            )
-
-            return feature_states
-
-        except requests.RequestException as e:
-            self.logger.warning(
-                f"Failed to fetch board defaults from {features_txt_url}: {e}"
-            )
-            return None
-
     @staticmethod
     def get_singleton():
         return APSourceMetadataFetcher.__singleton

metadata_manager/firmware_server/__init__.py

@@ -1,8 +1,11 @@
+from .features import FeaturesTxtClient
 from .manifest import ManifestJSON
-from .models import ReleaseRecord
+from .models import BoardArtifact, ReleaseRecord
 from .exceptions import ManifestFetchError
 
 __all__ = [
+    "BoardArtifact",
+    "FeaturesTxtClient",
     "ManifestJSON",
     "ManifestFetchError",
     "ReleaseRecord",

metadata_manager/firmware_server/features.py

@@ -0,0 +1,117 @@
+import logging
+
+import dill
+import redis
+import requests
+
+FEATURES_CACHE_TTL_SEC = 86400
+
+
+def parse_features_txt(text: str) -> dict:
+    """Parse features.txt content into a define to state mapping."""
+    feature_states = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line[0] == "!":
+            feature_name, state = line[1:], 0
+        else:
+            feature_name, state = line, 1
+        feature_states[feature_name] = state
+    return feature_states
+
+
+class FeaturesTxtClient:
+    """Fetch and cache board default features from firmware-server features.txt."""
+
+    def __init__(
+        self,
+        redis_host: str = "localhost",
+        redis_port: str = "6379",
+        caching_enabled: bool = True,
+        timeout: int = 30,
+        user_agent: str = "CustomBuild/1.0",
+    ):
+        self.caching_enabled = caching_enabled
+        self.timeout = timeout
+        self.user_agent = user_agent
+        self.logger = logging.getLogger(__name__)
+        self._cache_key_prefix = "features-"
+
+        if self.caching_enabled:
+            self._redis_client = redis.Redis(
+                host=redis_host,
+                port=redis_port,
+                decode_responses=False,
+            )
+
+    def _cache_key(self, url: str) -> str:
+        return self._cache_key_prefix + url
+
+    def _get_from_cache(self, url: str) -> tuple[str | None, dict | None]:
+        if not self.caching_enabled:
+            return None, None
+        value = self._redis_client.get(self._cache_key(url))
+        if value is None:
+            return None, None
+        entry = dill.loads(value)
+        if not isinstance(entry, dict):
+            return None, None
+        etag = entry.get("etag")
+        defaults = entry.get("defaults")
+        if not etag or defaults is None:
+            return None, None
+        return etag, defaults
+
+    def _store_in_cache(self, url: str, etag: str, defaults: dict) -> None:
+        if not self.caching_enabled:
+            return
+        self._redis_client.set(
+            name=self._cache_key(url),
+            value=dill.dumps({"etag": etag, "defaults": defaults}),
+            ex=FEATURES_CACHE_TTL_SEC,
+        )
+
+    def get_defaults(self, url: str) -> dict | None:
+        """
+        Fetch and parse features.txt for the given URL.
+
+        Returns a mapping of feature define to state (1 enabled, 0 disabled),
+        or None if the fetch fails.
+        """
+        cached_etag, cached_defaults = self._get_from_cache(url)
+        headers = {
+            "User-Agent": self.user_agent,
+            "Accept-Encoding": "identity",
+        }
+        if cached_etag:
+            headers["If-None-Match"] = cached_etag
+
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=self.timeout,
+            )
+            if response.status_code == 304:
+                return cached_defaults
+            response.raise_for_status()
+            defaults = parse_features_txt(response.text)
+            enabled_count = sum(1 for state in defaults.values() if state)
+            disabled_count = len(defaults) - enabled_count
+            self.logger.info(
+                "Fetched board defaults from firmware server: "
+                "%d enabled, %d disabled",
+                enabled_count,
+                disabled_count,
+            )
+            etag = response.headers.get("ETag")
+            if etag:
+                self._store_in_cache(url, etag, defaults)
+            return defaults
+        except requests.RequestException as exc:
+            self.logger.warning(
+                "Failed to fetch board defaults from %s: %s", url, exc
+            )
+            return None