From 692610cf7bf7c444867b652c55905e0132d0a57a Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:52:08 +0200 Subject: [PATCH 01/19] refactor!: replace configs gem with registry index --- Gemfile | 8 +- Gemfile.lock | 55 ++-- app/web/api/v1/configs.rb | 4 +- app/web/catalog/merge.rb | 101 -------- app/web/config/local_config.rb | 19 +- app/web/registry/config.rb | 239 ++++++++++++++++++ app/web/registry/errors.rb | 26 ++ app/web/registry/index.rb | 191 ++++++++++++++ app/web/registry/local_catalog.rb | 75 ++++++ app/web/registry/store.rb | 91 +++++++ app/web/registry/sync.rb | 71 ++++++ public/openapi.yaml | 100 +------- .../official/configs/phys.org/weekly.yml | 23 ++ .../support.apple.com/en_gb_ht201222.yml | 32 +++ .../registries/official/manifest.json | 10 + .../configs/secret.example/private.yml | 15 ++ .../fixtures/registries/private/manifest.json | 9 + spec/fixtures/registries/registries.yml | 11 + spec/html2rss/web/api/v1_spec.rb | 5 +- spec/html2rss/web/local_config_spec.rb | 20 +- spec/html2rss/web/registry/index_spec.rb | 56 ++++ spec/html2rss/web/registry/sync_spec.rb | 26 ++ spec/support/registry.rb | 24 ++ 23 files changed, 945 insertions(+), 266 deletions(-) delete mode 100644 app/web/catalog/merge.rb create mode 100644 app/web/registry/config.rb create mode 100644 app/web/registry/errors.rb create mode 100644 app/web/registry/index.rb create mode 100644 app/web/registry/local_catalog.rb create mode 100644 app/web/registry/store.rb create mode 100644 app/web/registry/sync.rb create mode 100644 spec/fixtures/registries/official/configs/phys.org/weekly.yml create mode 100644 spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 spec/fixtures/registries/official/manifest.json create mode 100644 spec/fixtures/registries/private/configs/secret.example/private.yml create mode 100644 spec/fixtures/registries/private/manifest.json create mode 100644 spec/fixtures/registries/registries.yml create mode 100644 spec/html2rss/web/registry/index_spec.rb create mode 100644 spec/html2rss/web/registry/sync_spec.rb create mode 100644 spec/support/registry.rb diff --git a/Gemfile b/Gemfile index 5816b002d..25acc1cc7 100644 --- a/Gemfile +++ b/Gemfile @@ -4,13 +4,9 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', '~> 0.27' +gem 'html2rss', path: '../html2rss' +# gem 'html2rss', '~> 0.27' # gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' -gem 'html2rss-configs', github: 'html2rss/html2rss-configs' - -# Use these instead of the two above (uncomment them) when developing locally: -# gem 'html2rss', path: '../html2rss' -# gem 'html2rss-configs', path: '../html2rss-configs' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index c62a3ec6a..6d94da0d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,27 @@ -GIT - remote: https://github.com/html2rss/html2rss-configs - revision: 89f3604ac4c8ecadefd8ccda4da60dff42648cd5 +PATH + remote: ../html2rss specs: - html2rss-configs (0.2.0) - html2rss + html2rss (0.27.1) + addressable (~> 2.7) + brotli + dry-validation + faraday (> 2.0.1, < 3.0) + faraday-follow_redirects + faraday-gzip (~> 3) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + reverse_markdown (~> 3.0) + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk GEM remote: https://rubygems.org/ @@ -103,27 +121,6 @@ GEM net-http (~> 0.5) hana (1.3.7) hashdiff (1.2.1) - html2rss (0.27.1) - addressable (~> 2.7) - brotli - dry-validation - faraday (> 2.0.1, < 3.0) - faraday-follow_redirects - faraday-gzip (~> 3) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - reverse_markdown (~> 3.0) - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk i18n (1.15.2) concurrent-ruby (~> 1.0) io-console (0.9.2) @@ -319,8 +316,7 @@ PLATFORMS DEPENDENCIES base64 climate_control - html2rss (~> 0.27) - html2rss-configs! + html2rss! irb puma rack-cache @@ -377,8 +373,7 @@ CHECKSUMS faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.27.1) sha256=a13f9c0d47f4c40038fc3f4b4dd452051385c9e1bc84c921024b730b3f42cd41 - html2rss-configs (0.2.0) + html2rss (0.27.1) i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 diff --git a/app/web/api/v1/configs.rb b/app/web/api/v1/configs.rb index 900b9b9b9..827450308 100644 --- a/app/web/api/v1/configs.rb +++ b/app/web/api/v1/configs.rb @@ -17,7 +17,7 @@ def index(_router) entries, duration_ms = build_entries emit_success(entries.size, duration_ms) success_payload(entries) - rescue Html2rss::Configs::Catalog::MissingDirectoryTitle => error + rescue Html2rss::Registry::CatalogBuilder::MissingDirectoryTitle => error emit_failure(error) raise end @@ -26,7 +26,7 @@ def index(_router) def build_entries started = Process.clock_gettime(Process::CLOCK_MONOTONIC) - entries = Html2rss::Web::Catalog::Merge.call + entries = Registry::Index.current.catalog_rows duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round [entries, duration_ms] end diff --git a/app/web/catalog/merge.rb b/app/web/catalog/merge.rb deleted file mode 100644 index d7e6b7734..000000000 --- a/app/web/catalog/merge.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'html2rss/configs' - -module Html2rss - module Web - ## - # Merges embedded catalog entries with local feed configs for the public catalog API. - module Catalog - module Merge - STARTER_FEED_IDS = %w[ - microsoft.com/azure-products - phys.org/weekly - softwareleadweekly.com/issues - ].freeze - - module_function - - ## - # @return [Array Object}>] - def call - embedded = Html2rss::Configs::Catalog.entries.map(&:to_h) - local = local_entries - (embedded + local).sort_by { |entry| entry.fetch(:id) } - end - - ## - # @return [Array Object}>] - def starter_entries - entries = call - selected = STARTER_FEED_IDS.filter_map { |id| entries.find { |entry| entry.fetch(:id) == id } } - selected.empty? ? entries.first(3) : selected - end - - ## - # @return [Array Object}>] - def local_entries - LocalConfig.feeds.filter_map do |feed_name, feed_config| - build_local_entry(feed_name, feed_config) - end - end - - ## - # @param feed_name [String, Symbol] - # @param feed_config [Hash] - # @return [Hash{Symbol => Object}, nil] - def build_local_entry(feed_name, feed_config) - directory = feed_config[:directory] || {} - title = directory[:title] - return nil if title.to_s.strip.empty? - - id = feed_name.to_s - channel = feed_config[:channel] || {} - - local_entry(id, directory, title, channel) - end - - ## - # @param id [String] - # @param directory [Hash] - # @param title [String] - # @param channel [Hash] - # @return [Hash{Symbol => Object}] - def local_entry(id, directory, title, channel) - { - id:, - path: "/#{id}.rss", - source: 'local', - directory: local_directory(directory, title), - channel: local_channel(channel, title), - parameters: { schema: {}, defaults: {} } - } - end - - ## - # @param directory [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def local_directory(directory, title) - { - title: title.to_s, - summary: directory[:summary], - topics: Array(directory[:topics]) - }.compact - end - - ## - # @param channel [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def local_channel(channel, title) - { - url: channel.fetch(:url), - language: channel[:language], - title: channel[:title] || title.to_s - }.compact - end - end - end - end -end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index fcf4f98d3..604643d00 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -3,12 +3,6 @@ require 'erb' require 'yaml' require_relative 'runtime_env' -begin - require 'html2rss/configs' -rescue LoadError => error - warn "[html2rss-web] Failed to load 'html2rss/configs': #{error.message}" - raise -end module Html2rss module Web @@ -28,7 +22,6 @@ class NotFound < RuntimeError; end # raised when the local config shape is invalid class InvalidConfig < RuntimeError; end FEED_EXTENSION_PATTERN = /\.(json|rss|xml)\z/ - EMBEDDED_FEED_NAME_PATTERN = %r{\A[^/]+/.+\z} # Path to local feed configuration file. CONFIG_FILE = 'config/feeds.yml' @@ -39,7 +32,7 @@ class << self # @return [Hash] def find(name) normalized_name = normalize_name(name) - config_hash = local_feed_config(normalized_name) || embedded_feed_config(normalized_name) + config_hash = local_feed_config(normalized_name) || registry_feed_config(normalized_name) raise NotFound, "Did not find local feed config at '#{normalized_name}'" unless config_hash config_hash @@ -94,6 +87,7 @@ def load_snapshot # @return [nil] def reload!(reason: 'manual') @mutex.synchronize { @snapshot = nil } + Registry::Index.reload! Observability.emit( event_name: 'cache.lifecycle', outcome: 'success', @@ -115,13 +109,8 @@ def local_feed_config(normalized_name) # @param normalized_name [String] # @return [Hash{Symbol=>Object}, nil] - def embedded_feed_config(normalized_name) - return nil unless defined?(Html2rss::Configs) - return nil unless normalized_name.match?(EMBEDDED_FEED_NAME_PATTERN) - - deep_dup(Html2rss::Configs.find_by_name(normalized_name)) - rescue Html2rss::Configs::ConfigNotFound - nil + def registry_feed_config(normalized_name) + Registry::Index.current.config_for(normalized_name) end # @param name [String, Symbol, #to_s] diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb new file mode 100644 index 000000000..06f693554 --- /dev/null +++ b/app/web/registry/config.rb @@ -0,0 +1,239 @@ +# frozen_string_literal: true + +require 'yaml' +require 'openssl' + +module Html2rss + module Web + module Registry + ## + # Registry definition parsed from {Config::REGISTRIES_FILE}. + Entry = Data.define( + :id, + :mode, + :path, + :sync_channel, + :sync_url, + :catalog, + :public_key_id, + :public_key + ) do + ## + # @return [Hash{String => OpenSSL::PKey::PKey}] + def public_keys + return {} if public_key.nil? + + { public_key_id => public_key } + end + end + + ## + # Parses registry configuration and applies zero-config defaults. + module Config # rubocop:disable Metrics/ModuleLength + REGISTRIES_FILE = 'config/registries.yml' + DEFAULT_PRECEDENCE = %w[official].freeze + DEFAULT_OFFICIAL_SYNC_CHANNEL = 'html2rss-official' + OFFICIAL_RELEASE_URL = 'https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz' + + @mutex = Mutex.new + @current = nil + + class << self # rubocop:disable Metrics/ClassLength + ## + # @return [Array] registry ids in merge precedence order + def precedence + current.precedence + end + + ## + # @param registry_id [String, Symbol] + # @return [Entry] + def entry(registry_id) + current.entries.fetch(registry_id.to_s) do + raise Errors::UnknownRegistry, "Unknown registry '#{registry_id}'" + end + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def catalog_enabled?(registry_id) + entry(registry_id).catalog + end + + ## + # @return [ConfigSnapshot] + def current + @mutex.synchronize { @current ||= parse_snapshot } + end + + ## + # Clears memoized configuration (tests and development reload). + # + # @return [nil] + def reload! + @mutex.synchronize { @current = nil } + nil + end + + private + + ## + # @return [ConfigSnapshot] + def parse_snapshot + document = load_document + precedence = Array(document[:precedence]).map(&:to_s) + precedence = DEFAULT_PRECEDENCE if precedence.empty? + + entries = precedence.to_h { |registry_id| [registry_id, parse_entry(registry_id, document)] } + missing = precedence - entries.keys + raise Errors::ConfigError, "Missing registry definitions: #{missing.join(', ')}" unless missing.empty? + + ConfigSnapshot.new(precedence:, entries:) + end + + ## + # @return [Hash{Symbol => Object}] + def load_document + path = registries_file + return default_document unless File.file?(path) + + YAML.safe_load_file(path, symbolize_names: true) || {} + rescue Psych::SyntaxError => error + raise Errors::ConfigError, "Invalid #{path}: #{error.message}" + end + + ## + # @return [String] + def registries_file + ENV.fetch('REGISTRIES_CONFIG', REGISTRIES_FILE) + end + + ## + # @return [Hash{Symbol => Object}] + def default_document + { + precedence: DEFAULT_PRECEDENCE, + registries: { + 'official' => { + sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, + catalog: true + } + } + } + end + + ## + # @param registry_id [String] + # @param document [Hash{Symbol => Object}] + # @return [Entry] + def parse_entry(registry_id, document) + raw = document.dig(:registries, registry_id.to_sym) || + document.dig(:registries, registry_id) + raise Errors::ConfigError, "Missing registry definition for '#{registry_id}'" unless raw.is_a?(Hash) + + build_entry(registry_id, raw) + end + + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @return [Entry] + def build_entry(registry_id, raw) + sync = sync_section(raw) + mode, resolved_path, sync_channel, sync_url = resolve_mode( + path: raw[:path]&.to_s, + sync_channel: sync[:channel]&.to_s, + sync_url: sync[:url]&.to_s + ) + + entry_attributes(registry_id, raw, mode, resolved_path, sync_channel, sync_url) + end + + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @param mode [Symbol] + # @param path [String, nil] + # @param sync_channel [String, nil] + # @param sync_url [String, nil] + # @return [Entry] + def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists + Entry.new( + id: registry_id, + mode:, + path:, + sync_channel:, + sync_url:, + catalog: raw.fetch(:catalog, true), + public_key_id: public_key_id_for(raw[:public_key_id]&.to_s), + public_key: parse_public_key(raw[:public_key]) + ) + end + + ## + # @param raw [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}] + def sync_section(raw) + raw[:sync].is_a?(Hash) ? raw[:sync] : {} + end + + ## + # @param public_key_id [String, nil] + # @return [String] + def public_key_id_for(public_key_id) + return public_key_id if public_key_id && !public_key_id.empty? + + 'official-v1' + end + + ## + # @param path [String, nil] + # @param sync_channel [String, nil] + # @param sync_url [String, nil] + # @return [Array(Symbol, String, nil, String, nil)] + def resolve_mode(path:, sync_channel:, sync_url:) + return [:path, expand_path(path), nil, nil] if path && !path.empty? + + resolved_sync_url = sync_url && !sync_url.empty? ? sync_url : resolve_sync_channel_url(sync_channel) + [:sync, nil, sync_channel, resolved_sync_url] + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_sync_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : DEFAULT_OFFICIAL_SYNC_CHANNEL + return OFFICIAL_RELEASE_URL if channel == DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param path [String] + # @return [String] + def expand_path(path) + return path if path.start_with?('/') + + File.expand_path(path, Dir.pwd) + end + + ## + # @param value [String, nil] + # @return [OpenSSL::PKey::PKey, nil] + def parse_public_key(value) + return nil if value.to_s.strip.empty? + + OpenSSL::PKey.read(value) + rescue OpenSSL::PKey::PKeyError => error + raise Errors::ConfigError, "Invalid registry public_key: #{error.message}" + end + end # rubocop:enable Metrics/ClassLength + + ## + # Parsed registry configuration snapshot. + ConfigSnapshot = Data.define(:precedence, :entries) + end # rubocop:enable Metrics/ModuleLength + end + end +end diff --git a/app/web/registry/errors.rb b/app/web/registry/errors.rb new file mode 100644 index 000000000..bb917c753 --- /dev/null +++ b/app/web/registry/errors.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Actionable registry runtime errors for operators and API handlers. + module Errors + # Base error for web registry operations. + class Error < StandardError; end + + # Raised when registry configuration is invalid or incomplete. + class ConfigError < Error; end + + # Raised when a registry bundle cannot be loaded. + class LoadError < Error; end + + # Raised when registry synchronization fails. + class SyncError < Error; end + + # Raised when a registry id is unknown. + class UnknownRegistry < Error; end + end + end + end +end diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb new file mode 100644 index 000000000..9132d728f --- /dev/null +++ b/app/web/registry/index.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Sole merge owner for registry bundles and local {LocalConfig} feeds. + class Index # rubocop:disable Metrics/ClassLength + RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) + StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) + + @mutex = Mutex.new + @current = nil + + class << self + ## + # @return [Index] + def current + @mutex.synchronize { @current ||= new } + end + + ## + # Clears memoized bundles (tests and development reload). + # + # @return [nil] + def reload! + @mutex.synchronize { @current = nil } + Config.reload! + nil + end + end + + ## + # @param feed_id [String, Symbol] + # @return [Hash{Symbol => Object}, nil] + def config_for(feed_id) + normalized_id = normalize_feed_id(feed_id) + local_config = local_config_for(normalized_id) + return local_config if local_config + + loaded_bundles.each_value do |bundle| + config = bundle.configs[normalized_id] + return deep_dup(config) if config + end + + nil + end + + ## + # @return [Array Object}>] catalog rows in HTTP wire shape + def catalog_rows + rows = registry_catalog_rows + LocalCatalog.rows.each { |row| rows[row.fetch(:id)] = row } + rows.values.sort_by { |row| row.fetch(:id) } + end + + ## + # @return [Array] + def status + Config.precedence.map do |registry_id| + entry = Config.entry(registry_id) + bundle = loaded_bundles[registry_id] + StatusEntry.new( + id: registry_id, + version: bundle&.manifest&.version, + updated_at: bundle_updated_at(registry_id), + sync_mode: entry.mode + ) + end + end + + private + + ## + # @return [Hash{String => Hash{Symbol => Object}}] + def registry_catalog_rows + Config.precedence.each_with_object({}) do |registry_id, rows| + next unless Config.catalog_enabled?(registry_id) + + bundle = loaded_bundles[registry_id] + next unless bundle + + bundle.catalog_entries.each do |entry| + rows[entry.id] ||= wire_registry_row(entry, registry_id) + end + end + end + + ## + # @return [Hash{String => RegistryBundle}] + def loaded_bundles + @loaded_bundles ||= Config.precedence.to_h { |registry_id| [registry_id, load_bundle(registry_id)] }.compact + end + + ## + # @param registry_id [String] + # @return [RegistryBundle, nil] + def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength + entry = Config.entry(registry_id) + directory = bundle_directory(entry) + return nil unless directory && File.directory?(directory) + + bundle = Html2rss::Registry::Bundle.load( + directory, + trust: entry.mode == :path ? :integrity_only : :signed, + public_keys: entry.public_keys + ) + to_registry_bundle(registry_id, bundle) + rescue Html2rss::Registry::Error => error + raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" + end + + ## + # @param registry_id [String] + # @param bundle [Html2rss::Registry::Bundle::BundleData] + # @return [RegistryBundle] + def to_registry_bundle(registry_id, bundle) + RegistryBundle.new( + registry_id:, + manifest: bundle.manifest, + configs: bundle.configs, + catalog_entries: bundle.catalog_entries + ) + end + + ## + # @param entry [Entry] + # @return [String, nil] + def bundle_directory(entry) + return entry.path if entry.mode == :path + + Store.registry_dir(entry.id) + end + + ## + # @param feed_id [String, Symbol] + # @return [String] + def normalize_feed_id(feed_id) + feed_id.to_s.delete_prefix('/').sub(LocalConfig::FEED_EXTENSION_PATTERN, '') + end + + ## + # @param feed_id [String] + # @return [Hash{Symbol => Object}, nil] + def local_config_for(feed_id) + feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] + return nil unless feed + + deep_dup(feed) + rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound + nil + end + + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [Hash{Symbol => Object}] + def wire_registry_row(entry, registry_id) + entry.to_h.merge(source: 'registry', registry: registry_id) + end + + ## + # @param registry_id [String] + # @return [Time, nil] + def bundle_updated_at(registry_id) + path = bundle_directory(Config.entry(registry_id)) + return nil unless path && File.directory?(path) + + manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + return nil unless File.file?(manifest_path) + + File.mtime(manifest_path) + end + + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) + case value + when Hash + value.transform_values { |entry| deep_dup(entry) } + when Array + value.map { |entry| deep_dup(entry) } + else + value + end + end + end # rubocop:enable Metrics/ClassLength + end + end +end diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb new file mode 100644 index 000000000..063dd7b29 --- /dev/null +++ b/app/web/registry/local_catalog.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Builds catalog wire rows for local feeds.yml entries. + module LocalCatalog + module_function + + ## + # @return [Array Object}>] + def rows + LocalConfig.feeds.filter_map do |feed_name, feed_config| + build_row(feed_name, feed_config) + end + end + + ## + # @param feed_name [String, Symbol] + # @param feed_config [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}, nil] + def build_row(feed_name, feed_config) + directory = feed_config[:directory] || {} + title = directory[:title] + return nil if title.to_s.strip.empty? + + row_identity(feed_name, directory, title, feed_config[:channel] || {}) + end + + ## + # @param feed_name [String, Symbol] + # @param directory [Hash{Symbol => Object}] + # @param title [String] + # @param channel [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}] + def row_identity(feed_name, directory, title, channel) + id = feed_name.to_s + { + id:, + path: "/#{id}.rss", + source: 'local', + directory: directory_payload(directory, title), + channel: channel_payload(channel, title), + parameters: { schema: {}, defaults: {} } + } + end + + ## + # @param directory [Hash{Symbol => Object}] + # @param title [String] + # @return [Hash{Symbol => Object}] + def directory_payload(directory, title) + { + title: title.to_s, + summary: directory[:summary], + topics: Array(directory[:topics]) + }.compact + end + + ## + # @param channel [Hash{Symbol => Object}] + # @param title [String] + # @return [Hash{Symbol => Object}] + def channel_payload(channel, title) + { + url: channel.fetch(:url), + language: channel[:language], + title: channel[:title] || title.to_s + }.compact + end + end + end + end +end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb new file mode 100644 index 000000000..523b89931 --- /dev/null +++ b/app/web/registry/store.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'fileutils' + +module Html2rss + module Web + module Registry + ## + # Manages on-disk registry bundle directories under {data_root}. + module Store + DEFAULT_DATA_ROOT = 'tmp/registry-data' + + class << self + ## + # @return [String] root directory for extracted registry bundles + def data_root + File.expand_path(ENV.fetch('REGISTRY_DATA_ROOT', DEFAULT_DATA_ROOT)) + end + + ## + # @param registry_id [String, Symbol] + # @return [String] active bundle directory for a registry id + def registry_dir(registry_id) + File.join(data_root, registry_id.to_s) + end + + ## + # Atomically replaces the active bundle directory for a registry id. + # + # @param registry_id [String, Symbol] + # @param staged_dir [String] verified bundle directory to promote + # @return [String] promoted bundle directory + def swap!(registry_id, staged_dir) + raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) + + target = registry_dir(registry_id) + parent = File.dirname(target) + FileUtils.mkdir_p(parent) + + backup = "#{target}.backup.#{Process.pid}" + promote_bundle!(staged_dir, target, backup) + target + end + + ## + # Copies a seed bundle into the data root when no active bundle exists. + # + # @param registry_id [String, Symbol] + # @param seed_path [String] bundle directory to copy + # @return [Boolean] true when a seed copy was performed + def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod + target = registry_dir(registry_id) + return false if bundle_present?(target) + + raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) + + FileUtils.mkdir_p(File.dirname(target)) + FileUtils.cp_r(seed_path, target) + true + end + + private + + ## + # @param staged_dir [String] + # @param target [String] + # @param backup [String] + # @return [void] + def promote_bundle!(staged_dir, target, backup) + FileUtils.rm_rf(backup) + FileUtils.mv(target, backup) if File.exist?(target) + FileUtils.mv(staged_dir, target) + rescue StandardError + FileUtils.rm_rf(target) + FileUtils.mv(backup, target) if File.exist?(backup) + raise + ensure + FileUtils.rm_rf(backup) + end + + ## + # @param path [String] + # @return [Boolean] + def bundle_present?(path) + File.directory?(path) && !Dir.empty?(path) + end + end + end + end + end +end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb new file mode 100644 index 000000000..fb7188bee --- /dev/null +++ b/app/web/registry/sync.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Fetches and stores signed registry bundles (network sync is expanded in Phase 4). + module Sync + OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL + + SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) + + class << self + ## + # Resolves the download URL for a sync-mode registry id. + # + # @param registry_id [String, Symbol] + # @return [String] + def sync_url_for(registry_id) + entry = Config.entry(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync + + entry.sync_url + end + + ## + # Runs synchronization for a registry id. + # + # Phase 3 provides a minimal stub; Phase 4 adds fetch, verify, and store. + # + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def run(registry_id:) + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" + end + + raise Errors::SyncError, 'Registry network sync is not enabled until Phase 4' + end + + ## + # @param registry_id [String, Symbol, nil] + # @return [Array] + def status(registry_id: nil) + rows = Index.current.status + rows = rows.select { |row| row.id == registry_id.to_s } if registry_id + rows.map { |row| sync_status_for(row) } + end + + private + + ## + # @param row [Index::StatusEntry] + # @return [SyncStatus] + def sync_status_for(row) + entry = Config.entry(row.id) + SyncStatus.new( + registry_id: row.id, + mode: row.sync_mode, + version: row.version, + updated_at: row.updated_at, + sync_url: entry.sync_url, + last_error: nil + ) + end + end + end + end + end +end diff --git a/public/openapi.yaml b/public/openapi.yaml index c0d64312e..aa22ceea4 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -132,101 +132,10 @@ paths: parameters: properties: defaults: - properties: - blog: - type: - - string - - 'null' - id: - type: - - string - - 'null' - region: - type: - - string - - 'null' - repository: - type: - - string - - 'null' - section: - type: - - string - - 'null' - user_id: - type: - - string - - 'null' - username: - type: - - string - - 'null' + properties: {} type: object schema: - properties: - blog: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - id: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - region: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - repository: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - section: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - user_id: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - username: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' + properties: {} type: object required: - schema @@ -234,15 +143,18 @@ paths: type: object path: type: string + registry: + type: string source: type: string required: - id - path - - source - directory - channel - parameters + - source + - registry type: object type: array required: diff --git a/spec/fixtures/registries/official/configs/phys.org/weekly.yml b/spec/fixtures/registries/official/configs/phys.org/weekly.yml new file mode 100644 index 000000000..763f70f2b --- /dev/null +++ b/spec/fixtures/registries/official/configs/phys.org/weekly.yml @@ -0,0 +1,23 @@ +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..76afeafbe --- /dev/null +++ b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,32 @@ +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/spec/fixtures/registries/official/manifest.json b/spec/fixtures/registries/official/manifest.json new file mode 100644 index 000000000..17d99eece --- /dev/null +++ b/spec/fixtures/registries/official/manifest.json @@ -0,0 +1,10 @@ +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test", + "files": { + "configs/phys.org/weekly.yml": "0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373", + "configs/support.apple.com/en_gb_ht201222.yml": "3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac" + } +} diff --git a/spec/fixtures/registries/private/configs/secret.example/private.yml b/spec/fixtures/registries/private/configs/secret.example/private.yml new file mode 100644 index 000000000..6b516f6c2 --- /dev/null +++ b/spec/fixtures/registries/private/configs/secret.example/private.yml @@ -0,0 +1,15 @@ +directory: + title: "Private Corp Feed" + summary: "Hidden from catalog." +channel: + title: "Private Corp Feed" + url: https://secret.example/private + ttl: 60 +selectors: + items: + selector: article + title: + selector: h1 + url: + selector: a + extractor: href diff --git a/spec/fixtures/registries/private/manifest.json b/spec/fixtures/registries/private/manifest.json new file mode 100644 index 000000000..0aacc3ef5 --- /dev/null +++ b/spec/fixtures/registries/private/manifest.json @@ -0,0 +1,9 @@ +{ + "format": "registry.v1", + "registry_id": "private", + "version": "test-private", + "public_key_id": "test", + "files": { + "configs/secret.example/private.yml": "196760640262a019c0b32e08d2f5272c08b11b10b5880dc13266922a30c872a3" + } +} diff --git a/spec/fixtures/registries/registries.yml b/spec/fixtures/registries/registries.yml new file mode 100644 index 000000000..b505e5794 --- /dev/null +++ b/spec/fixtures/registries/registries.yml @@ -0,0 +1,11 @@ +precedence: + - official + - private + +registries: + official: + path: spec/fixtures/registries/official + catalog: true + private: + path: spec/fixtures/registries/private + catalog: false diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index e037715f7..4d505cebf 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -221,7 +221,10 @@ def relative_feed_link_header(token) json = expect_success_response(last_response) expect(json.dig('meta', 'catalog_version')).to eq(1) expect(json.dig('data', 'configs')).to be_an(Array) - expect(json.dig('data', 'configs').first).to include('id', 'path', 'source', 'directory', 'channel', 'parameters') + first = json.dig('data', 'configs').first + expect(first).to include('id', 'path', 'source', 'directory', 'channel', 'parameters') + expect(first['source']).to eq('registry') + expect(first).to include('registry' => 'official') end it 'returns 404 when the catalog is disabled', :aggregate_failures do diff --git a/spec/html2rss/web/local_config_spec.rb b/spec/html2rss/web/local_config_spec.rb index aa0a10d35..dfb6f9b44 100644 --- a/spec/html2rss/web/local_config_spec.rb +++ b/spec/html2rss/web/local_config_spec.rb @@ -51,33 +51,19 @@ def account_token(snapshot) expect(titles_for('example.json', 'example.rss', 'example.xml')).to eq(%w[Example Example Example]) end - it 'falls back to embedded configs when the feed is not in local yaml' do - stub_const('Html2rss::Configs', Module.new do - def self.find_by_name(_name); end - end) - stub_const('Html2rss::Configs::ConfigNotFound', Class.new(StandardError)) - allow(Html2rss::Configs) - .to receive(:find_by_name) - .with('support.apple.com/en_gb_ht201222') - .and_return({ channel: { title: 'Apple security releases' } }) + it 'falls back to registry configs when the feed is not in local yaml' do allow(described_class).to receive(:snapshot).and_return(empty_snapshot) config = described_class.find('support.apple.com/en_gb_ht201222.rss') - expect(config).to include(channel: { title: 'Apple security releases' }) + expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) end - it 'returns not found for malformed embedded config paths instead of depending on gem error messages' do - stub_const('Html2rss::Configs', Module.new do - def self.find_by_name(_name); end - end) - stub_const('Html2rss::Configs::ConfigNotFound', Class.new(StandardError)) - allow(Html2rss::Configs).to receive(:find_by_name) + it 'returns not found for unknown feed ids' do allow(described_class).to receive(:snapshot).and_return(empty_snapshot) expect { described_class.find('/broken-name.rss') } .to raise_error(described_class::NotFound, "Did not find local feed config at 'broken-name'") - expect(Html2rss::Configs).not_to have_received(:find_by_name) end end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb new file mode 100644 index 000000000..3b9cf095b --- /dev/null +++ b/spec/html2rss/web/registry/index_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Index do + describe '#config_for' do + it 'returns registry configs by feed id' do + config = described_class.current.config_for('support.apple.com/en_gb_ht201222') + + expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) + end + + it 'returns nil for unknown ids' do + expect(described_class.current.config_for('missing.example/feed')).to be_nil + end + + it 'still resolves catalog-disabled registry feeds by id' do + config = described_class.current.config_for('secret.example/private') + + expect(config).to include(channel: hash_including(url: 'https://secret.example/private')) + end + end + + describe '#catalog_rows' do + it 'includes registry rows with source and registry fields' do + rows = described_class.current.catalog_rows + phys = rows.find { |row| row.fetch(:id) == 'phys.org/weekly' } + + expect(phys).to include( + source: 'registry', + registry: 'official', + path: '/phys.org/weekly.rss' + ) + end + + it 'omits catalog-disabled registries from the catalog API rows' do + rows = described_class.current.catalog_rows + + expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') + end + end + + describe '#status' do + it 'reports loaded registry metadata' do + status = described_class.current.status + official = status.find { |entry| entry.id == 'official' } + + expect(official).to have_attributes( + version: 'test-fixture', + sync_mode: :path + ) + end + end +end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb new file mode 100644 index 000000000..8c7703475 --- /dev/null +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Sync do + describe '.sync_url_for' do + it 'resolves the official release URL for sync-mode defaults' do + ClimateControl.modify('REGISTRIES_CONFIG' => nil) do + Html2rss::Web::Registry::Index.reload! + + expect(described_class.sync_url_for('official')).to eq( + Html2rss::Web::Registry::Config::OFFICIAL_RELEASE_URL + ) + end + end + end + + describe '.run' do + it 'rejects path-mode registries' do + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /path mode/) + end + end +end diff --git a/spec/support/registry.rb b/spec/support/registry.rb new file mode 100644 index 000000000..c07e52949 --- /dev/null +++ b/spec/support/registry.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module RegistryTestHelpers + FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) + DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') + + module_function + + def configure_registry_fixtures! + ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') + end + + def reset_registry! + Html2rss::Web::Registry::Index.reload! + end +end + +RSpec.configure do |config| + config.before do + RegistryTestHelpers.configure_registry_fixtures! + RegistryTestHelpers.reset_registry! + end +end From c485cecf30c1676341f22019b7fbe0fb3513f8f3 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:56:02 +0200 Subject: [PATCH 02/19] feat(registry): sync CLI, seed bundle, and background refresh --- Dockerfile | 2 + app/registries/seed/official/.gitkeep | 0 app/web/api/v1/root_metadata.rb | 40 +- app/web/boot/setup.rb | 6 + app/web/registry/config.rb | 2 +- app/web/registry/index.rb | 2 +- app/web/registry/store.rb | 65 ++- app/web/registry/sync.rb | 378 +++++++++++++++++- app/web/security/security_logger.rb | 7 + bin/docker-build | 1 + bin/prepare-registry-seed | 31 ++ bin/registry-sync | 60 +++ config/registries.yml | 11 + docker-compose.yml | 6 + public/openapi.yaml | 21 +- .../sync/bundle/configs/phys.org/weekly.yml | 23 ++ .../support.apple.com/en_gb_ht201222.yml | 32 ++ .../registries/sync/bundle/manifest.json | 1 + spec/fixtures/registries/sync/registries.yml | 13 + spec/html2rss/web/api/v1_spec.rb | 15 + spec/html2rss/web/boot/setup_spec.rb | 2 + spec/html2rss/web/registry/sync_spec.rb | 93 +++++ spec/support/registry_sync.rb | 64 +++ 23 files changed, 851 insertions(+), 24 deletions(-) create mode 100644 app/registries/seed/official/.gitkeep create mode 100755 bin/prepare-registry-seed create mode 100755 bin/registry-sync create mode 100644 config/registries.yml create mode 100644 spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml create mode 100644 spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 spec/fixtures/registries/sync/bundle/manifest.json create mode 100644 spec/fixtures/registries/sync/registries.yml create mode 100644 spec/support/registry_sync.rb diff --git a/Dockerfile b/Dockerfile index d9bc8e0a5..0d91ae5cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ RUN apk add --no-cache \ && mkdir -p /app \ && mkdir -p /app/tmp/rack-cache-body \ && mkdir -p /app/tmp/rack-cache-meta \ + && mkdir -p /app/data/registries \ && chown "$USER":"$USER" -R /app WORKDIR /app @@ -89,6 +90,7 @@ COPY --chown=$USER:$USER bin/docker-healthcheck ./bin/docker-healthcheck COPY --chown=$USER:$USER Gemfile Gemfile.lock app.rb config.ru ./ COPY --chown=$USER:$USER app ./app COPY --chown=$USER:$USER config ./config +COPY --chown=$USER:$USER app/registries/seed ./app/registries/seed COPY --chown=$USER:$USER public ./public COPY --from=frontend-builder --chown=$USER:$USER /app/frontend/dist ./frontend/dist diff --git a/app/registries/seed/official/.gitkeep b/app/registries/seed/official/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 562f5a86b..01a989068 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -27,16 +27,40 @@ def build(router) # @return [Hash{Symbol=>Object}] def instance_payload(router) { - feed_creation: { - enabled: Flags.auto_source_enabled?, - access_token_required: Flags.auto_source_enabled? - }, - catalog: { - enabled: Flags.config_catalog_enabled?, - url: "#{router.base_url}/api/v1/configs" - } + feed_creation: feed_creation_payload, + catalog: catalog_payload(router), + registries: registry_status_rows + } + end + + # @return [Hash{Symbol => Object}] + def feed_creation_payload + { + enabled: Flags.auto_source_enabled?, + access_token_required: Flags.auto_source_enabled? + } + end + + # @param router [Roda::RodaRequest] + # @return [Hash{Symbol => Object}] + def catalog_payload(router) + { + enabled: Flags.config_catalog_enabled?, + url: "#{router.base_url}/api/v1/configs" } end + + # @return [Array Object}>] + def registry_status_rows + Registry::Index.current.status.map do |entry| + { + id: entry.id, + version: entry.version, + updated_at: entry.updated_at&.utc&.iso8601, + sync_mode: entry.sync_mode.to_s + } + end + end end end end diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index 55ac05b34..d41be5449 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -24,6 +24,7 @@ def call! configure_request_service! configure_runtime_logging! configure_gem_defaults! + configure_registry! log_startup! end @@ -76,6 +77,11 @@ def configure_runtime_logging! Rack::Timeout::Logger.logger = AppLogger.logger end + # @return [void] + def configure_registry! + Registry::Sync.boot! + end + # @return [void] def log_startup! AppLogger.logger.info( diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 06f693554..4fd5b8842 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -184,7 +184,7 @@ def sync_section(raw) def public_key_id_for(public_key_id) return public_key_id if public_key_id && !public_key_id.empty? - 'official-v1' + 'html2rss:registry:2026' end ## diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 9132d728f..6052c4bf4 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -102,7 +102,7 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength bundle = Html2rss::Registry::Bundle.load( directory, - trust: entry.mode == :path ? :integrity_only : :signed, + trust: :integrity_only, public_keys: entry.public_keys ) to_registry_bundle(registry_id, bundle) diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 523b89931..fc9e7459e 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'fileutils' +require 'json' module Html2rss module Web @@ -9,6 +10,8 @@ module Registry # Manages on-disk registry bundle directories under {data_root}. module Store DEFAULT_DATA_ROOT = 'tmp/registry-data' + DEFAULT_SEED_ROOT = '/app/registries/seed' + SYNC_STATE_FILE = '.sync-state.json' class << self ## @@ -17,6 +20,12 @@ def data_root File.expand_path(ENV.fetch('REGISTRY_DATA_ROOT', DEFAULT_DATA_ROOT)) end + ## + # @return [String] root directory for image-shipped seed bundles + def seed_root + File.expand_path(ENV.fetch('REGISTRY_SEED_ROOT', DEFAULT_SEED_ROOT)) + end + ## # @param registry_id [String, Symbol] # @return [String] active bundle directory for a registry id @@ -24,6 +33,20 @@ def registry_dir(registry_id) File.join(data_root, registry_id.to_s) end + ## + # @param registry_id [String, Symbol] + # @return [String] seed bundle directory for a registry id + def seed_path_for(registry_id) + File.join(seed_root, registry_id.to_s) + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def bundle_present?(registry_id) + bundle_present_at?(registry_dir(registry_id)) + end + ## # Atomically replaces the active bundle directory for a registry id. # @@ -50,7 +73,7 @@ def swap!(registry_id, staged_dir) # @return [Boolean] true when a seed copy was performed def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod target = registry_dir(registry_id) - return false if bundle_present?(target) + return false if bundle_present?(registry_id) raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) @@ -59,8 +82,46 @@ def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMe true end + ## + # @param registry_id [String, Symbol] + # @return [Hash{Symbol => Object}] + def sync_state(registry_id) + read_sync_state.fetch(registry_id.to_s, {}) + end + + ## + # @param registry_id [String, Symbol] + # @param last_error [String, nil] + # @param last_sync_at [Time, nil] + # @return [void] + def write_sync_state!(registry_id, last_error:, last_sync_at: Time.now.utc) + state = read_sync_state + state[registry_id.to_s] = { + 'last_error' => last_error, + 'last_sync_at' => last_sync_at&.iso8601 + }.compact + FileUtils.mkdir_p(data_root) + File.write(sync_state_path, JSON.generate(state)) + end + private + ## + # @return [String] + def sync_state_path + File.join(data_root, SYNC_STATE_FILE) + end + + ## + # @return [Hash{String => Hash{String => Object}}] + def read_sync_state + return {} unless File.file?(sync_state_path) + + JSON.parse(File.read(sync_state_path)) + rescue JSON::ParserError + {} + end + ## # @param staged_dir [String] # @param target [String] @@ -81,7 +142,7 @@ def promote_bundle!(staged_dir, target, backup) ## # @param path [String] # @return [Boolean] - def bundle_present?(path) + def bundle_present_at?(path) File.directory?(path) && !Dir.empty?(path) end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index fb7188bee..4d7f30cf9 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -1,16 +1,39 @@ # frozen_string_literal: true +require 'fileutils' +require 'net/http' +require 'json' +require 'stringio' +require 'uri' + module Html2rss module Web module Registry ## - # Fetches and stores signed registry bundles (network sync is expanded in Phase 4). - module Sync + # Fetches, verifies, and stores signed registry bundles. + module Sync # rubocop:disable Metrics/ModuleLength OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + + DEFAULT_ALLOWED_HOSTS = %w[ + api.github.com + github.com + objects.githubusercontent.com + ].freeze + FETCH_OPEN_TIMEOUT_SECONDS = 10 + FETCH_READ_TIMEOUT_SECONDS = 60 + MAX_RESPONSE_BYTES = Html2rss::Registry::Archive::MAX_TARBALL_BYTES + BACKGROUND_JITTER_FRACTION = 0.1 SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) - class << self + @boot_mutex = Mutex.new + @boot_started = false + @timer_started = false + + class << self # rubocop:disable Metrics/ClassLength ## # Resolves the download URL for a sync-mode registry id. # @@ -20,23 +43,35 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - entry.sync_url + resolve_download_url(entry) end ## # Runs synchronization for a registry id. # - # Phase 3 provides a minimal stub; Phase 4 adds fetch, verify, and store. - # # @param registry_id [String, Symbol] + # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] - def run(registry_id:) + def run(registry_id:, dry_run: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength entry = Config.entry(registry_id) if entry.mode == :path raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" end - raise Errors::SyncError, 'Registry network sync is not enabled until Phase 4' + staging_root = nil + download_url = resolve_download_url(entry) + staging_root, staged_dir = fetch_and_verify!(entry, download_url) + unless dry_run + Store.swap!(registry_id, staged_dir) + Index.reload! + record_success!(registry_id) + end + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + rescue StandardError => error + record_failure!(registry_id, error) unless dry_run + raise + ensure + FileUtils.rm_rf(staging_root) if staging_root end ## @@ -48,24 +83,345 @@ def status(registry_id: nil) rows.map { |row| sync_status_for(row) } end + ## + # Seeds sync-mode registries and optionally syncs on boot. + # + # @return [void] + def boot! # rubocop:disable Metrics/MethodLength + @boot_mutex.synchronize do + return if @boot_started + + @boot_started = true + end + return if skip_boot? + + Config.precedence.each do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + + seed_registry!(registry_id) + schedule_boot_sync!(registry_id) if boot_sync?(registry_id) + end + + start_background_timer! + end + + ## + # Starts a jittered background sync loop when enabled. + # + # @return [void] + def start_background_timer! # rubocop:disable Metrics/MethodLength + interval_hours = Integer(ENV.fetch('REGISTRY_SYNC_INTERVAL_HOURS', '24')) + return if interval_hours <= 0 + + @boot_mutex.synchronize do + return if @timer_started + + @timer_started = true + end + + Thread.new do # rubocop:disable ThreadSafety/NewThread -- background registry refresh by design + sleep(background_jitter_seconds(interval_hours)) + loop do + sync_all! + sleep(interval_hours * 3600) + end + end + end + + ## + # @return [Integer] process exit code for CLI use (0 ok, 1 when sync registries lack bundles) + def cli_exit_code + unusable_sync_registries.empty? ? 0 : 1 + end + + ## + # @return [Array] sync-mode registry ids without a usable on-disk bundle + def unusable_sync_registries + Config.precedence.filter_map do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + next if Store.bundle_present?(registry_id) + + registry_id + end + end + private + ## + # @return [Boolean] + def skip_boot? + ENV.fetch('RACK_ENV', 'development') == 'test' + end + + ## + # @param registry_id [String] + # @return [void] + def seed_registry!(registry_id) # rubocop:disable Metrics/MethodLength + seed_path = Store.seed_path_for(registry_id) + return unless File.directory?(seed_path) + + seeded = Store.seed_if_empty!(registry_id, seed_path:) + Index.reload! if seeded + rescue Errors::LoadError => error + AppLogger.logger.warn( + { + component: 'registry', + event_name: 'registry.seed', + outcome: 'failure', + registry_id:, + error: error.message + }.to_json + ) + end + + ## + # @param registry_id [String] + # @return [Boolean] + def boot_sync?(registry_id) + ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' || !Store.bundle_present?(registry_id) + end + + ## + # @param registry_id [String] + # @return [void] + def schedule_boot_sync!(registry_id) + Thread.new { run(registry_id:) } # rubocop:disable ThreadSafety/NewThread -- non-blocking first boot + rescue StandardError + nil + end + + ## + # @return [void] + def sync_all! + Config.precedence.each do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + + run(registry_id:) + rescue StandardError + nil + end + end + + ## + # @param interval_hours [Integer] + # @return [Numeric] + def background_jitter_seconds(interval_hours) + max_jitter = [(interval_hours * 3600 * BACKGROUND_JITTER_FRACTION).to_i, 1].max + rand(max_jitter) + end + + ## + # @param entry [Entry] + # @return [String] + def resolve_download_url(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + return resolve_official_download_url if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = Fetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @param entry [Entry] + # @param download_url [String] + # @return [Array(String, String)] staging root and verified bundle directory + def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + tarball = Fetcher.fetch!(download_url) + staging_root = Dir.mktmpdir('registry-sync-') + staged_dir = File.join(staging_root, 'bundle') + FileUtils.mkdir_p(staged_dir) + + StringIO.open(tarball) do |io| + Html2rss::Registry::Archive.extract!(io, into: staged_dir) + end + + Html2rss::Registry::Verifier.verify!( + staged_dir, + trust: :signed, + public_keys: entry.public_keys + ) + [staging_root, staged_dir] + rescue Html2rss::Registry::VerificationError => error + log_signature_failure!(entry.id, error.message) if signature_failure?(error) + raise Errors::SyncError, error.message + rescue Html2rss::Registry::ArchiveError => error + raise Errors::SyncError, error.message + end + + ## + # @param error [Html2rss::Registry::VerificationError] + # @return [Boolean] + def signature_failure?(error) + error.message.match?(/signature|public_key_id/i) + end + + ## + # @param registry_id [String] + # @param message [String] + # @return [void] + def log_signature_failure!(registry_id, message) + SecurityLogger.log_registry_signature_failure(registry_id, message) + end + + ## + # @param registry_id [String] + # @return [void] + def record_success!(registry_id) + Store.write_sync_state!(registry_id, last_error: nil) + row = Index.current.status.find { |entry| entry.id == registry_id.to_s } + Observability.emit( + event_name: 'registry.sync', + outcome: 'success', + details: { registry_id:, version: row&.version } + ) + end + + ## + # @param registry_id [String] + # @param error [StandardError] + # @return [void] + def record_failure!(registry_id, error) + Store.write_sync_state!(registry_id, last_error: error.message) + Observability.emit( + event_name: 'registry.sync', + outcome: 'failure', + details: { registry_id:, error: error.message }, + level: :warn + ) + end + ## # @param row [Index::StatusEntry] # @return [SyncStatus] def sync_status_for(row) entry = Config.entry(row.id) + state = Store.sync_state(row.id) SyncStatus.new( registry_id: row.id, mode: row.sync_mode, version: row.version, updated_at: row.updated_at, - sync_url: entry.sync_url, - last_error: nil + sync_url: entry.mode == :sync ? resolve_download_url(entry) : nil, + last_error: state['last_error'] ) end + end # rubocop:enable Metrics/ClassLength + + ## + # HTTPS fetcher with host allowlisting and redirect rejection. + module Fetcher + module_function + + ## + # @param url [String] + # @return [String] response body + def fetch!(url) + uri = parse_https_uri!(url) + ensure_allowed_host!(uri.host) + perform_request!(uri) + end + + ## + # @param url [String] + # @return [URI::HTTPS] + def parse_https_uri!(url) + uri = URI(url) + unless uri.is_a?(URI::HTTPS) + raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" + end + + uri + end + + ## + # @param host [String] + # @return [void] + def ensure_allowed_host!(host) + allowed = DEFAULT_ALLOWED_HOSTS + extra_allowed_hosts + return if allowed.include?(host) + + raise Errors::SyncError, "Registry sync host not allowed: #{host}" + end + + ## + # @return [Array] + def extra_allowed_hosts + ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + .split(',') + .map(&:strip) + .reject(&:empty?) + end + + ## + # @param uri [URI::HTTPS] + # @return [String] + def perform_request!(uri) # rubocop:disable Metrics/MethodLength + response = nil + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: true, + open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, + read_timeout: FETCH_READ_TIMEOUT_SECONDS + ) do |http| + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/octet-stream' + request['User-Agent'] = 'html2rss-web/registry-sync' + response = http.request(request) + end + + reject_redirect!(response) + reject_error_status!(response) + read_body!(response) + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_redirect!(response) + return unless response.is_a?(Net::HTTPRedirection) + + raise Errors::SyncError, 'Registry sync rejects HTTP redirects' + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_error_status!(response) + return if response.is_a?(Net::HTTPSuccess) + + raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" + end + + ## + # @param response [Net::HTTPResponse] + # @return [String] + def read_body!(response) + body = response.body.to_s + if body.bytesize > MAX_RESPONSE_BYTES + raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" + end + + body + end end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/security/security_logger.rb b/app/web/security/security_logger.rb index 1349e8155..e6293584b 100644 --- a/app/web/security/security_logger.rb +++ b/app/web/security/security_logger.rb @@ -69,6 +69,13 @@ def log_blocked_request(ip, reason, endpoint) log_event('blocked_request', { ip:, reason:, endpoint: }, severity: :warn) end + # @param registry_id [String] + # @param reason [String] + # @return [void] + def log_registry_signature_failure(registry_id, reason) + log_event('registry_signature_failure', { registry_id:, reason: }, severity: :warn) + end + private # @param event_type [String] diff --git a/bin/docker-build b/bin/docker-build index 7b4e5f9d7..7fd9be1c9 100755 --- a/bin/docker-build +++ b/bin/docker-build @@ -1,4 +1,5 @@ #!/bin/bash set -eux +"$(dirname "$0")/prepare-registry-seed" docker build --no-cache -t html2rss/web -f Dockerfile . diff --git a/bin/prepare-registry-seed b/bin/prepare-registry-seed new file mode 100755 index 000000000..a2c38dd59 --- /dev/null +++ b/bin/prepare-registry-seed @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# frozen_string_literal: true + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +CONFIGS_REPO="${HTML2RSS_CONFIGS_ROOT:-$ROOT_DIR/../html2rss-configs}" +SEED_DIR="$ROOT_DIR/app/registries/seed/official" +OUTPUT_TAR="$(mktemp -t registry-seed.XXXXXX.tar.gz)" + +cleanup() { + rm -f "$OUTPUT_TAR" +} +trap cleanup EXIT + +if [[ ! -d "$CONFIGS_REPO/configs" ]]; then + echo "Missing configs tree at $CONFIGS_REPO/configs" >&2 + exit 1 +fi + +echo "Building registry seed bundle from $CONFIGS_REPO" +( + cd "$CONFIGS_REPO" + BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" +) + +rm -rf "$SEED_DIR" +mkdir -p "$SEED_DIR" +tar -xzf "$OUTPUT_TAR" -C "$SEED_DIR" + +echo "Seed bundle prepared at $SEED_DIR" diff --git a/bin/registry-sync b/bin/registry-sync new file mode 100755 index 000000000..1c013a099 --- /dev/null +++ b/bin/registry-sync @@ -0,0 +1,60 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'optparse' + +require_relative '../app' + +module RegistrySyncCLI + module_function + + def run(argv = ARGV) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + options = { status: false, registry_id: nil, dry_run: false } + OptionParser.new do |opts| + opts.banner = 'Usage: bin/registry-sync [options]' + opts.on('--status', 'Print registry sync status and exit') { options[:status] = true } + opts.on('--registry ID', 'Sync a single registry id') { |value| options[:registry_id] = value } + opts.on('--dry-run', 'Fetch and verify without swapping the active bundle') { options[:dry_run] = true } + end.parse!(argv) + + if options[:status] + print_status_table(options[:registry_id]) + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + + target_registry_ids(options[:registry_id]).each do |registry_id| + Html2rss::Web::Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) + end + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + + def target_registry_ids(registry_id) + return Array(registry_id) if registry_id + + Html2rss::Web::Registry::Config.precedence + end + + def print_status_table(registry_id) # rubocop:disable Metrics/MethodLength + rows = Html2rss::Web::Registry::Sync.status(registry_id:) + headers = %w[registry mode version updated_at sync_url last_error] + puts headers.join("\t") + rows.each do |row| + puts [ + row.registry_id, + row.mode, + row.version || '-', + format_time(row.updated_at), + row.sync_url || '-', + row.last_error || '-' + ].join("\t") + end + end + + def format_time(value) + return '-' unless value + + value.utc.iso8601 + end +end + +RegistrySyncCLI.run if $PROGRAM_NAME == __FILE__ diff --git a/config/registries.yml b/config/registries.yml new file mode 100644 index 000000000..3ebeca8e5 --- /dev/null +++ b/config/registries.yml @@ -0,0 +1,11 @@ +precedence: + - official + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + # Pin the publisher Ed25519 public key for signed network sync. + # Seed bundles copied from the image use integrity-only verification. diff --git a/docker-compose.yml b/docker-compose.yml index c098fb039..6ca4cc89c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: RACK_TIMEOUT_SERVICE_TIMEOUT: 55 BOTASAURUS_SCRAPE_TIMEOUT_SECONDS: 45 BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS: 30 + REGISTRY_DATA_ROOT: /app/data/registries BOTASAURUS_SCRAPER_URL: http://botasaurus:4010 # Trial runs use the image's bundled config/feeds.yml. # Uncomment the block below when you want to replace it with your own file. @@ -32,6 +33,8 @@ services: # source: ./config/feeds.yml # target: /app/config/feeds.yml # read_only: true + volumes: + - registry-data:/app/data/registries watchtower: image: containrrr/watchtower @@ -53,3 +56,6 @@ services: SCRAPE_WORK_TIMEOUT_SECONDS: 30 ports: - "127.0.0.1:4010:4010" + +volumes: + registry-data: diff --git a/public/openapi.yaml b/public/openapi.yaml index aa22ceea4..22991265b 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -64,9 +64,28 @@ paths: - enabled - access_token_required type: object + registries: + items: + properties: + id: + type: string + sync_mode: + type: string + updated_at: + type: string + version: + type: string + required: + - id + - version + - updated_at + - sync_mode + type: object + type: array required: - feed_creation - catalog + - registries type: object required: - api @@ -78,7 +97,7 @@ paths: - success - data type: object - description: returns catalog pointer metadata + description: returns registry status metadata security: - {} summary: API metadata diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml new file mode 100644 index 000000000..763f70f2b --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml @@ -0,0 +1,23 @@ +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..76afeafbe --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,32 @@ +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/spec/fixtures/registries/sync/bundle/manifest.json b/spec/fixtures/registries/sync/bundle/manifest.json new file mode 100644 index 000000000..e74570c9a --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/manifest.json @@ -0,0 +1 @@ +{"format":"registry.v1","registry_id":"official","version":"test-fixture","public_key_id":"test-key","files":{"configs/phys.org/weekly.yml":"0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373","configs/support.apple.com/en_gb_ht201222.yml":"3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac"}} \ No newline at end of file diff --git a/spec/fixtures/registries/sync/registries.yml b/spec/fixtures/registries/sync/registries.yml new file mode 100644 index 000000000..e307db29a --- /dev/null +++ b/spec/fixtures/registries/sync/registries.yml @@ -0,0 +1,13 @@ +precedence: + - official + +registries: + official: + sync: + url: https://registry.test.example/registry-bundle.tar.gz + catalog: true + public_key_id: test-key + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index 4d505cebf..5eb375b23 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -196,6 +196,21 @@ def relative_feed_link_header(token) ) end + it 'returns registry status metadata', :aggregate_failures do + get '/api/v1' + + expect(last_response.status).to eq(200) + json = expect_success_response(last_response) + official = json.dig('data', 'instance', 'registries').find { |row| row['id'] == 'official' } + + expect(official).to include( + 'id' => 'official', + 'version' => 'test-fixture', + 'sync_mode' => 'path' + ) + expect(official['updated_at']).to be_a(String) + end + it 'returns API information with trailing slash', :aggregate_failures do get '/api/v1/' diff --git a/spec/html2rss/web/boot/setup_spec.rb b/spec/html2rss/web/boot/setup_spec.rb index 116b60811..d98558843 100644 --- a/spec/html2rss/web/boot/setup_spec.rb +++ b/spec/html2rss/web/boot/setup_spec.rb @@ -21,6 +21,7 @@ before do allow(Html2rss::Web::Flags).to receive(:validate!) allow(Html2rss::Web::Boot::Sentry).to receive(:configure!) + allow(Html2rss::Web::Registry::Sync).to receive(:boot!) end describe '.call!' do @@ -33,6 +34,7 @@ expect(Html2rss::Web::EnvironmentValidator).to have_received(:validate_environment!).once expect(Html2rss::Web::EnvironmentValidator).to have_received(:validate_production_security!).once expect(Html2rss::Web::Flags).to have_received(:validate!).once + expect(Html2rss::Web::Registry::Sync).to have_received(:boot!).once end it 'routes rack-timeout logs through the shared app logger' do diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 8c7703475..4ff1b8877 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'spec_helper' +require 'webmock/rspec' require_relative '../../../../app' @@ -17,10 +18,102 @@ end end + describe '.run', :registry_sync do + let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } + let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } + + before do + stub_request(:get, download_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + end + + it 'fetches, verifies, and stores a signed bundle', :aggregate_failures do + status = described_class.run(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + expect(Html2rss::Web::Registry::Index.current.config_for('phys.org/weekly')).to include( + channel: hash_including(title: 'Phys.org — Weekly') + ) + end + + it 'supports dry-run verification without swapping the active bundle' do + expect do + described_class.run(registry_id: 'official', dry_run: true) + end.not_to(change { Html2rss::Web::Registry::Store.bundle_present?('official') }) + end + + it 'rejects redirects' do + stub_request(:get, download_url) + .to_return(status: 302, headers: { 'Location' => 'https://evil.example/bundle.tar.gz' }) + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect/i) + end + + it 'logs signature verification failures to the security logger' do + allow(Html2rss::Web::SecurityLogger).to receive(:log_registry_signature_failure) + stub_request(:get, download_url).to_return(status: 200, body: 'not-a-tarball') + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError) + + expect(Html2rss::Web::SecurityLogger).not_to have_received(:log_registry_signature_failure) + end + + it 'keeps the previous bundle when sync fails' do + described_class.run(registry_id: 'official') + stub_request(:get, download_url).to_return(status: 500, body: 'fail') + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /HTTP 500/) + + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + end + end + describe '.run' do it 'rejects path-mode registries' do expect { described_class.run(registry_id: 'official') } .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /path mode/) end end + + describe '.status', :registry_sync do + it 'includes sync metadata and last error state' do + row = described_class.status(registry_id: 'official').first + + expect(row).to have_attributes( + registry_id: 'official', + mode: :sync, + sync_url: 'https://registry.test.example/registry-bundle.tar.gz' + ) + end + end + + describe '.cli_exit_code', :registry_sync do + it 'returns non-zero when a sync registry has no bundle' do + expect(described_class.cli_exit_code).to eq(1) + end + + it 'returns zero after a successful sync' do + stub_request(:get, 'https://registry.test.example/registry-bundle.tar.gz') + .to_return(status: 200, body: RegistrySyncTestHelpers.build_signed_tarball) + + described_class.run(registry_id: 'official') + + expect(described_class.cli_exit_code).to eq(0) + end + end + + describe '.boot!' do + it 'does not run in the test environment' do + allow(described_class).to receive(:seed_registry!) + allow(described_class).to receive(:start_background_timer!) + + described_class.boot! + + expect(described_class).not_to have_received(:seed_registry!) + end + end end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb new file mode 100644 index 000000000..b0f6a1dca --- /dev/null +++ b/spec/support/registry_sync.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'openssl' +require 'fileutils' +require 'zlib' +require 'stringio' +require 'rubygems/package' + +module RegistrySyncTestHelpers + HTML2RSS_FIXTURES = File.expand_path('../../../html2rss/spec/fixtures/registry', __dir__) + TEST_PUBLIC_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pub')) + TEST_PRIVATE_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pem')) + SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) + SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') + SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') + + module_function + + def configure_sync_registry! + ENV['REGISTRIES_CONFIG'] = SYNC_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = SYNC_DATA_ROOT + ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example' + FileUtils.rm_rf(SYNC_DATA_ROOT) + Html2rss::Web::Registry::Index.reload! + end + + def build_signed_tarball # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + bundle_dir = Dir.mktmpdir('signed-registry-bundle') + FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) + manifest = Html2rss::Registry::Manifest.parse( + File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + private_key = OpenSSL::PKey.read(TEST_PRIVATE_KEY) + signature = private_key.sign(nil, manifest.canonical_bytes) + File.write( + File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), + [signature].pack('m0') + ) + + tar_io = StringIO.new + Gem::Package::TarWriter.new(tar_io) do |tar| + Dir.glob(File.join(bundle_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| + next if File.directory?(path) + + relative = path.delete_prefix("#{bundle_dir}/") + tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } + end + end + + gz_io = StringIO.new + Zlib::GzipWriter.wrap(gz_io) { |gz| gz.write(tar_io.string) } + gz_io.string + ensure + FileUtils.rm_rf(bundle_dir) + end +end + +RSpec.configure do |config| + config.before do |example| + next unless example.metadata[:registry_sync] + + RegistrySyncTestHelpers.configure_sync_registry! + end +end From 668a3e2eb79e1a4853a0feae23513b2272dba07b Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:57:20 +0200 Subject: [PATCH 03/19] docs: registry sync runbook and catalog API ownership --- AGENTS.md | 11 ++++--- docs/README.md | 88 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 07b31e815..80d1bdb08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,19 +56,22 @@ See [docs/design-system.md](docs/design-system.md) for visual rules. ## Config catalog API -Public feed-directory metadata for embedded and local configs. +Public feed-directory metadata from verified registry bundles and local `feeds.yml` entries. | Item | Detail | | --- | --- | | Endpoint | `GET /api/v1/configs` | | Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | | Disabled response | `404` with `{ "error": "catalog_disabled" }` | -| Embedded entries | `Html2rss::Configs::Catalog.entries` — do not re-walk YAML in the handler | -| Local entries | `Catalog::Merge` includes `feeds.yml` feeds only when `directory.title` is set | +| Registry entries | `Registry::Index.current.catalog_rows` — loads signed bundles from `config/registries.yml`; adds `source: registry`, `registry: ` | +| Local entries | `Registry::LocalCatalog` includes `feeds.yml` feeds only when `directory.title` is set (`source: local`) | +| Per-registry privacy | `catalog: false` in `registries.yml` omits that registry from the API (feeds still served) | | Starter feeds (UI) | Frontend `selectStarterFeeds` when feed creation is disabled; catalog find uses full catalog when enabled | | Catalog find | `findCatalogEntries` → multi-hit list under create URL; links via `catalogFeedHref` (path + defaults) | | CORS | Route-scoped on `/api/v1/configs` only (`GET`, `OPTIONS`) | -| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` | +| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` and `instance.registries` sync status | | Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | +Registry sync: `bin/registry-sync --status`; boot seed + optional sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). + After handler or envelope changes: `make openapi` and `make ci-ready`. diff --git a/docs/README.md b/docs/README.md index 480eb526b..0cd53fcb1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,7 +26,7 @@ Welcome! This is the canonical source of truth for contributing to `html2rss-web - **Runtime behavior**: Application code plus tests. - **HTTP contract**: Request specs plus generated OpenAPI. -- **Config catalog API**: `GET /api/v1/configs` — embedded data from `Html2rss::Configs::Catalog`, merged with local `feeds.yml` entries that include `directory.title`. Disabled when `CONFIG_CATALOG_ENABLED=false` (`404`, `catalog_disabled`). CORS is enabled on this route only. +- **Config catalog API**: `GET /api/v1/configs` — catalog rows from `Registry::Index` (verified registry bundles per `config/registries.yml`, merged with local `feeds.yml` entries that include `directory.title`). Disabled when `CONFIG_CATALOG_ENABLED=false` (`404`, `catalog_disabled`). CORS is enabled on this route only. - **This file**: Contributor conventions and current project rules. --- @@ -284,6 +284,92 @@ Tune alert thresholds from sustained `request.error` or `feed.render` failure sp --- +## Registry sync runbook + +Signed feed registries replace the embedded `html2rss-configs` gem. Each registry is defined in `config/registries.yml` (override path with `REGISTRIES_CONFIG`). + +### Check sync status + +Inside the Dev Container: + +```bash +bin/registry-sync --status +``` + +Columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any registry reports `last_error`. + +Sync a single registry or dry-run without swapping the active bundle: + +```bash +bin/registry-sync --registry official +bin/registry-sync --registry official --dry-run +``` + +### Boot behavior + +`Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): + +1. **Seed** — copies the image-embedded official bundle when no on-disk bundle exists. +2. **Sync on boot** — when `REGISTRY_SYNC_ON_BOOT=true`, or when the bundle is missing, fetches and verifies the latest release. +3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` is greater than zero (default `24`), re-syncs on a jittered timer. Set to `0` to disable. + +Network sync verifies Ed25519 signatures using the `public_key` pinned in `registries.yml`. Seed bundles from the Docker image use integrity-only verification. + +### Add a corporate registry + +```yaml +precedence: + - official + - corp + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- + + corp: + sync: + url: https://registry.example.com/registry-bundle.tar.gz + catalog: false # feeds served; omitted from GET /api/v1/configs + public_key_id: corp:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +- **`precedence`** — merge order for feed lookup; first match wins. +- **`sync.url`** — direct tarball URL, or use `sync.channel: html2rss-official` for the default GitHub release asset. +- **`catalog: false`** — private registry: configs are served at `/{registry.id}.rss` but excluded from the public catalog API (privacy for internal feeds). +- Restrict outbound hosts with `REGISTRY_SYNC_ALLOWED_HOSTS` (comma-separated hostnames). + +### Air-gapped / offline path mount + +For environments without outbound network access, mount a verified bundle directory: + +```yaml +registries: + official: + path: /opt/html2rss/registry/official + catalog: true +``` + +The directory must contain `manifest.json`, optional `manifest.sig`, and `configs/`. Path mode skips network sync; run `bin/registry-sync --status` to confirm `mode` is `path`. + +### Key rotation + +1. Publish a new bundle signed with the new key (`public_key_id` in `manifest.json`). +2. Update `public_key_id` and `public_key` in `registries.yml` on every instance **before** or together with the first release that requires the new key. +3. Re-sync (`bin/registry-sync` or wait for background refresh). Failed verification leaves the previous bundle active and records `last_error`. + +--- + ## Documentation Policy - Prefer deleting stale docs over archiving them in-place. From d6fefb42b0b2b7cd61f9bc9eb1c8e545ea5415b9 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 15:21:32 +0200 Subject: [PATCH 04/19] docs: add registry go-live manual --- docs/README.md | 2 + docs/registry-go-live.md | 486 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 docs/registry-go-live.md diff --git a/docs/README.md b/docs/README.md index 0cd53fcb1..0fcf1b7f2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -286,6 +286,8 @@ Tune alert thresholds from sustained `request.error` or `feed.render` failure sp ## Registry sync runbook +For end-to-end release and deployment steps (maintainers and operators), see [registry-go-live.md](registry-go-live.md). + Signed feed registries replace the embedded `html2rss-configs` gem. Each registry is defined in `config/registries.yml` (override path with `REGISTRIES_CONFIG`). ### Check sync status diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md new file mode 100644 index 000000000..c79a907d4 --- /dev/null +++ b/docs/registry-go-live.md @@ -0,0 +1,486 @@ +# Registry go-live manual + +Step-by-step guide to ship and operate signed `registry.v1` bundles across the html2rss org repos. + +**Related docs** + +- Bundle format: [`html2rss/docs/registry-v1.md`](../../html2rss/docs/registry-v1.md) +- Operator runbook (sync flags, custom registries): [README.md — Registry sync runbook](./README.md#registry-sync-runbook) + +--- + +## 1. Maintainers — merge and release order + +Work in this order when a change touches registry contracts or feed configs. + +### 1.1 `html2rss` (core gem) + +Source of truth for `registry.v1` schema, `Html2rss::Registry::Verifier`, `CatalogBuilder`, and archive limits. + +1. Merge registry-related changes to `main`. +2. Regenerate schema if config schema changed: + ```bash + cd html2rss + mise exec -- make ready + ``` +3. Tag/release the gem when the contract is stable (web depends on this). + +### 1.2 `html2rss-configs` (signed bundle publisher) + +Source of truth for feed YAML and signed release artifacts. + +1. Merge config changes to `master`. +2. Run the configs quality gate: + ```bash + cd html2rss-configs + make ready + ``` +3. Tag the release (tag name becomes `manifest.json` `version` in CI via `REGISTRY_VERSION: ${{ github.ref_name }}`): + ```bash + git tag 2026.08.22 # example; use your release tag + git push origin 2026.08.22 + ``` +4. Tag push triggers [`.github/workflows/release.yml`](../../html2rss-configs/.github/workflows/release.yml): + - `make registry-build -- --sign` + - uploads `dist/registry-bundle.tar.gz` to the GitHub Release + +### 1.3 `html2rss-web` (runtime + Docker image) + +Consumes verified bundles; ships an unsigned seed copy in the image. + +1. Bump the `html2rss` gem dependency if the core contract changed. +2. Update `config/registries.yml` when the signing key or sync channel changes (see sections 2 and 5). +3. Build the image with a fresh seed (section 3). +4. Merge to `main`, wait for CI (`ci` workflow) to pass. +5. Publish a GitHub Release on `html2rss-web` — [`.github/workflows/release.yml`](../../html2rss-web/.github/workflows/release.yml) builds and pushes `html2rss/web` tags (`latest`, semver, major, commit SHA). + +**Rule of thumb:** core contract → configs signed release → web image that embeds/verifies that release. + +--- + +## 2. First signed release — signing key, tag, verify artifact + +### 2.1 Generate an Ed25519 key pair + +Use OpenSSL (same algorithm as `tool/registry-build` and `Html2rss::Registry::Verifier`): + +```bash +openssl genpkey -algorithm ED25519 -out registry-signing.pem +openssl pkey -in registry-signing.pem -pubout -out registry-signing.pub +``` + +Keep the private key offline except where signing happens. + +### 2.2 Configure `html2rss-configs` CI secret + +1. In the `html2rss-configs` GitHub repo, add secret **`REGISTRY_SIGNING_KEY`** with the full PEM contents of `registry-signing.pem`. +2. Release CI reads it here: + + ```yaml + env: + REGISTRY_VERSION: ${{ github.ref_name }} + REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} + run: make registry-build -- --sign + ``` + +### 2.3 Pin the public key in `html2rss-web` + +Add the public key to `config/registries.yml` before instances need to sync signed bundles over the network: + +```yaml +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +- `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). +- Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). + +### 2.4 Tag and publish the configs release + +```bash +cd html2rss-configs +git tag # e.g. 2026.08.22 +git push origin +``` + +Wait for the Release workflow to finish. The asset name is always **`registry-bundle.tar.gz`**. + +### 2.5 Verify the release artifact + +Download the asset from the GitHub Release, then verify locally: + +```bash +mkdir -p /tmp/registry-verify && tar -xzf registry-bundle.tar.gz -C /tmp/registry-verify + +# Inspect manifest +jq . /tmp/registry-verify/manifest.json +# Confirm: format=registry.v1, registry_id=official, version=, public_key_id matches pin + +# Confirm signature file exists +test -f /tmp/registry-verify/manifest.sig + +# Optional: verify with core gem (from html2rss checkout) +cd html2rss +mise exec -- bundle exec ruby -rhtml2rss -ropenssl -e " + pk = OpenSSL::PKey.read(File.read('path/to/registry-signing.pub')) + Html2rss::Registry::Verifier.verify!( + '/tmp/registry-verify', + trust: :signed, + public_keys: { 'html2rss:registry:2026' => pk } + ) + puts 'OK' +" +``` + +Unsigned local builds (no `--sign`) are valid for seed/integrity-only use only: + +```bash +cd html2rss-configs +make registry-build # writes dist/registry-bundle.tar.gz without manifest.sig +``` + +--- + +## 3. Web image build — seed preparation and Docker + +### 3.1 How seed gets into the image + +| Step | What happens | +| --- | --- | +| `bin/prepare-registry-seed` | Builds an **unsigned** bundle from `html2rss-configs` and extracts it to `app/registries/seed/official/` | +| `bin/docker-build` | Runs `prepare-registry-seed`, then `docker build --no-cache -t html2rss/web` | +| `Dockerfile` | `COPY app/registries/seed ./app/registries/seed` (image path `/app/registries/seed`) | +| Boot | `Registry::Sync.boot!` copies seed → `REGISTRY_DATA_ROOT/` when no on-disk bundle exists | + +`prepare-registry-seed` details: + +- Configs source: `HTML2RSS_CONFIGS_ROOT` (default: sibling `../html2rss-configs`) +- Runs `tool/registry-build` **without** `--sign` (integrity-only seed) +- Output directory: `app/registries/seed/official/` + +### 3.2 Build locally + +From a workspace with both repos checked out as siblings: + +```bash +cd html2rss-web +bin/docker-build +``` + +Or prepare seed only: + +```bash +cd html2rss-web +HTML2RSS_CONFIGS_ROOT=/path/to/html2rss-configs bin/prepare-registry-seed +docker build --no-cache -t html2rss/web -f Dockerfile . +``` + +Set build metadata for production parity: + +```bash +docker build \ + --build-arg BUILD_TAG=1.2.3 \ + --build-arg GIT_SHA="$(git rev-parse HEAD)" \ + -t html2rss/web \ + -f Dockerfile . +``` + +### 3.3 What operators get in the image + +- **`/app/registries/seed/official/`** — unsigned bundle baked at build time (offline-first bootstrap). +- **`/app/config/registries.yml`** — default official registry with `sync.channel: html2rss-official`. +- **`/app/data/registries`** — empty at build; populated at runtime (seed copy + network sync). + +--- + +## 4. Operators — default path (official registry) + +### 4.1 Pull a new image (simplest update path) + +`docker-compose.yml` defaults: + +- Image: `html2rss/web` +- Volume: `registry-data:/app/data/registries` (`REGISTRY_DATA_ROOT`) +- Watchtower optional: checks for new images every 7200s + +```bash +docker compose pull html2rss-web +docker compose up -d html2rss-web +``` + +A new image updates the **seed** inside the container. Existing data in the volume is kept until sync replaces it. + +### 4.2 Zero-config first boot + +With the stock `config/registries.yml`, no extra registry env vars are required. + +On boot (`Registry::Sync.boot!`): + +1. **Seed** — if `REGISTRY_DATA_ROOT/official/` is empty, copy from `/app/registries/seed/official/`. +2. **Sync on boot** — runs when `REGISTRY_SYNC_ON_BOOT=true` **or** when no bundle is present on disk (after seed attempt). +3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` > 0 (default **24**), re-sync on a jittered timer. Set to `0` to disable. + +Official sync URL (from `config/registries.yml` + `Registry::Config`): + +`https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz` + +Allowed outbound hosts (built-in): `api.github.com`, `github.com`, `objects.githubusercontent.com`. + +### 4.3 Existing instance — volume retained + +The named volume preserves synced bundles across container restarts and image upgrades. After pull + restart: + +- Old bundle stays active until a successful sync swaps it. +- Use `bin/registry-sync` or wait for background refresh to pick up a new configs release without rebuilding the image (section 7). + +--- + +## 5. Operators — custom corporate registry + +Override or extend `config/registries.yml` (or set `REGISTRIES_CONFIG` to an alternate file path). + +Example from the [registry sync runbook](./README.md#add-a-corporate-registry): + +```yaml +precedence: + - official + - corp + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- + + corp: + sync: + url: https://registry.example.com/registry-bundle.tar.gz + catalog: false + public_key_id: corp:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +| Field | Purpose | +| --- | --- | +| `precedence` | Feed lookup merge order; first match wins | +| `sync.url` | Direct HTTPS tarball URL for network sync | +| `sync.channel: html2rss-official` | Resolves to the official GitHub release asset | +| `catalog: false` | Feeds served at `/{feed_id}.rss`; **omitted** from `GET /api/v1/configs` | +| `public_key` / `public_key_id` | Required for `:signed` network sync verification | + +For hosts outside the default GitHub allowlist: + +```bash +REGISTRY_SYNC_ALLOWED_HOSTS=registry.example.com,cdn.example.com +``` + +Mount a custom registries file in Compose: + +```yaml +volumes: + - ./config/registries.yml:/app/config/registries.yml:ro +``` + +--- + +## 6. Verify live + +Run these checks after deploy or sync. + +### 6.1 Registry sync status (CLI) + +Inside the running container (or Dev Container): + +```bash +bin/registry-sync --status +``` + +Tab-separated columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. + +- Exit code **0** — all sync-mode registries have a usable on-disk bundle. +- Exit code **1** — at least one sync registry lacks a bundle (see `Registry::Sync.unusable_sync_registries`). + +Single registry or dry-run: + +```bash +bin/registry-sync --registry official +bin/registry-sync --registry official --dry-run +``` + +### 6.2 Instance metadata API + +```bash +curl -sS http://127.0.0.1:4000/api/v1/ | jq '.data.instance' +``` + +Confirm: + +- **`instance.registries`** — array with `id`, `version`, `updated_at`, `sync_mode` per configured registry +- **`instance.catalog`** — `{ "enabled": true, "url": ".../api/v1/configs" }` (unless `CONFIG_CATALOG_ENABLED=false`) + +### 6.3 Catalog API + +```bash +curl -sS http://127.0.0.1:4000/api/v1/configs | jq '.data.configs[0]' +``` + +Expect rows with `source: "registry"`, `registry: "official"`, and a `path` like `/anthropic.com/news.rss`. + +When `CONFIG_CATALOG_ENABLED=false`, expect `404` with `{ "error": "catalog_disabled" }`. + +### 6.4 Sample static feed + +Pick a feed id from the catalog (`id` field) or from `configs/.yml` in the bundle. Request: + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:4000/anthropic.com/news.rss +``` + +Expect HTTP **200** and valid RSS XML. + +--- + +## 7. Update configs without pulling a new Docker image + +Configs releases are independent of web image releases. To refresh feeds on a running instance: + +### 7.1 Manual sync + +```bash +bin/registry-sync --registry official +``` + +Fetches the latest signed tarball, verifies signature + digests, atomically swaps the active bundle, and reloads `Registry::Index`. + +### 7.2 Automatic refresh + +| Mechanism | Env var | Default | Behavior | +| --- | --- | --- | --- | +| Sync on boot | `REGISTRY_SYNC_ON_BOOT` | `false` | When `true`, network sync on every boot (in addition to missing-bundle sync) | +| Background timer | `REGISTRY_SYNC_INTERVAL_HOURS` | `24` | Jittered periodic re-sync; `0` disables | +| Missing bundle | — | — | Always syncs on boot when no on-disk bundle exists | + +Example — force sync every boot (use sparingly): + +```bash +REGISTRY_SYNC_ON_BOOT=true +``` + +Example — disable background refresh: + +```bash +REGISTRY_SYNC_INTERVAL_HOURS=0 +``` + +**Note:** Pulling a new web image updates the embedded seed but does **not** replace an existing volume bundle until sync succeeds. + +--- + +## 8. Troubleshooting + +### 8.1 Signature verification failure + +Symptoms in `bin/registry-sync --status`: + +- `last_error` contains `Unknown public_key_id`, `Invalid manifest signature`, or `Missing manifest.sig` + +Checks: + +1. `public_key_id` and `public_key` in `registries.yml` match the signed release. +2. Deploy web config **before** or **with** the first bundle signed by a new key (key rotation). +3. Downloaded asset is `registry-bundle.tar.gz` from the expected release (not an unsigned local build). + +Signature-related failures are logged via `SecurityLogger.log_registry_signature_failure`. + +Verify without swapping the active bundle: + +```bash +bin/registry-sync --registry official --dry-run +``` + +### 8.2 Sync failure keeps the old bundle + +By design: + +- `Registry::Sync.run` only calls `Store.swap!` after fetch **and** verification succeed. +- On failure, `last_error` is recorded; the previous bundle under `REGISTRY_DATA_ROOT//` remains served. +- `Store.promote_bundle!` rolls back on swap failure. + +If sync fails on first boot with no prior bundle, the instance may have only the unsigned seed (integrity-only) until a signed sync succeeds. + +### 8.3 Network / host errors + +| Error pattern | Likely cause | +| --- | --- | +| `Registry sync host not allowed` | Add host to `REGISTRY_SYNC_ALLOWED_HOSTS` | +| `Registry sync rejects HTTP redirects` | Publish a direct HTTPS asset URL (`sync.url`) | +| `Registry sync fetch failed with HTTP …` | Release missing, URL wrong, or GitHub outage | +| `Registry sync requires HTTPS URLs` | Use `https://` in `sync.url` | + +Default allowed hosts cover official GitHub release downloads only. + +### 8.4 Air-gapped / offline (`path` mode) + +For environments without outbound sync, mount a verified bundle directory and skip network sync: + +```yaml +registries: + official: + path: /opt/html2rss/registry/official + catalog: true +``` + +Requirements: + +- Directory contains `manifest.json`, `configs/`, and optionally `manifest.sig` +- `bin/registry-sync` is not applicable (`path mode; sync is not applicable`) +- `bin/registry-sync --status` should show `mode: path` + +Load path uses `:integrity_only` trust (disk/image trust boundary). + +In Docker Compose, bind-mount the bundle and optionally set `REGISTRIES_CONFIG`: + +```yaml +volumes: + - /opt/html2rss/registry/official:/opt/html2rss/registry/official:ro +environment: + REGISTRIES_CONFIG: /app/config/registries.yml +``` + +### 8.5 CLI exit code non-zero with empty `last_error` + +`bin/registry-sync --status` exits **1** when a sync-mode registry has **no on-disk bundle**. Run a sync or confirm seed copy succeeded: + +```bash +bin/registry-sync --registry official +bin/registry-sync --status +``` + +### 8.6 Seed missing or stale in image + +If `app/registries/seed/official/` was not populated before `docker build`, first boot may have nothing to seed. + +Fix for maintainers: + +```bash +cd html2rss-web +bin/prepare-registry-seed # or bin/docker-build +``` + +Ensure `HTML2RSS_CONFIGS_ROOT` points at the configs checkout used for the release. From 2a8e5f3c037a9423d976f194221acdcb03fd1ded Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 15:23:58 +0200 Subject: [PATCH 05/19] fix(deps): pin html2rss from GitHub for CI Replace sibling path pin with feat/registry-v1 branch so CI can bundle install without a checked-out html2rss repo. --- Gemfile | 4 ++-- Gemfile.lock | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index 25acc1cc7..ae3e430ef 100644 --- a/Gemfile +++ b/Gemfile @@ -4,9 +4,9 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', path: '../html2rss' +# gem 'html2rss', path: '../html2rss' # gem 'html2rss', '~> 0.27' -# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index 6d94da0d8..37331d84e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,5 +1,7 @@ -PATH - remote: ../html2rss +GIT + remote: https://github.com/html2rss/html2rss + revision: e32014ac089679cac7e91fd55f3bd85a0489771e + branch: feat/registry-v1 specs: html2rss (0.27.1) addressable (~> 2.7) @@ -143,7 +145,7 @@ GEM loofah (2.25.2) crass (~> 1.0.2) nokogiri (>= 1.12.0) - mcp (1.2.0) + mcp (1.3.0) json_schemer (>= 2.4) mime-types (3.7.0) logger @@ -384,7 +386,7 @@ CHECKSUMS lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 loofah (2.25.2) sha256=2007f746959ac65552456e04b433e83deb22759ab38c838b4445c70e43425918 - mcp (1.2.0) sha256=af75a270fbcbff5db74992e1d0664cfe7e2aa7f89fa9b31592bba40e9f554ba6 + mcp (1.3.0) sha256=9395aa3a054eb8986b7714ec8abb25b533729af896f4875da4c4cb7a3024fbae mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 From 36055c9b2c97f7b0b7d5cce4f933f8c525dfdd3b Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:22:05 +0200 Subject: [PATCH 06/19] feat(registry): sync governance with staged promote and trust context Add sync policy, staging/promote flow, trust context verification, scrape policy guards, and seed bundle fixtures with updated registry sync specs. --- Gemfile | 3 +- app/registries/seed/official/.gitkeep | 0 .../seed/official/configs/phys.org/weekly.yml | 25 ++ .../support.apple.com/en_gb_ht201222.yml | 34 ++ app/registries/seed/official/manifest.json | 10 + app/web/registry/catalog_change_reporter.rb | 96 ++++++ app/web/registry/config.rb | 135 ++++++-- app/web/registry/index.rb | 62 ++-- app/web/registry/local_catalog.rb | 10 +- app/web/registry/local_catalog_row.rb | 18 + app/web/registry/registry_catalog_row.rb | 35 ++ app/web/registry/scrape_policy.rb | 58 ++++ app/web/registry/store.rb | 99 +++++- app/web/registry/sync.rb | 318 ++++++++---------- app/web/registry/sync_coordinator.rb | 23 ++ app/web/registry/sync_fetcher.rb | 150 +++++++++ app/web/registry/sync_url_resolver.rb | 80 +++++ app/web/registry/trust_context.rb | 54 +++ app/web/registry/version_gate.rb | 40 +++ app/web/security/security_logger.rb | 7 + bin/registry-sync | 13 +- config/registries.yml | 11 + docs/README.md | 17 +- docs/registry-go-live.md | 76 ++++- frontend/src/api/generated/types.gen.ts | 39 +-- public/openapi.yaml | 1 - spec/fixtures/registries/keys/test-key.pem | 3 + spec/fixtures/registries/keys/test-key.pub | 3 + .../official/configs/phys.org/weekly.yml | 2 + .../support.apple.com/en_gb_ht201222.yml | 2 + .../registries/official/manifest.json | 4 +- .../configs/secret.example/private.yml | 2 + .../fixtures/registries/private/manifest.json | 4 +- .../sync/bundle/configs/phys.org/weekly.yml | 2 + .../support.apple.com/en_gb_ht201222.yml | 2 + .../registries/sync/bundle/manifest.json | 11 +- spec/fixtures/registries/sync/registries.yml | 1 + spec/html2rss/web/registry/config_spec.rb | 93 +++++ spec/html2rss/web/registry/index_spec.rb | 37 ++ .../web/registry/scrape_policy_spec.rb | 56 +++ spec/html2rss/web/registry/store_spec.rb | 80 +++++ spec/html2rss/web/registry/sync_spec.rb | 106 +++++- .../web/registry/sync_url_resolver_spec.rb | 39 +++ .../web/registry/version_gate_spec.rb | 20 ++ spec/smoke/docker_spec.rb | 9 + spec/spec_helper.rb | 14 +- spec/support/openapi.rb | 6 + spec/support/registry_sync.rb | 69 ++-- 48 files changed, 1648 insertions(+), 331 deletions(-) delete mode 100644 app/registries/seed/official/.gitkeep create mode 100644 app/registries/seed/official/configs/phys.org/weekly.yml create mode 100644 app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 app/registries/seed/official/manifest.json create mode 100644 app/web/registry/catalog_change_reporter.rb create mode 100644 app/web/registry/local_catalog_row.rb create mode 100644 app/web/registry/registry_catalog_row.rb create mode 100644 app/web/registry/scrape_policy.rb create mode 100644 app/web/registry/sync_coordinator.rb create mode 100644 app/web/registry/sync_fetcher.rb create mode 100644 app/web/registry/sync_url_resolver.rb create mode 100644 app/web/registry/trust_context.rb create mode 100644 app/web/registry/version_gate.rb create mode 100644 spec/fixtures/registries/keys/test-key.pem create mode 100644 spec/fixtures/registries/keys/test-key.pub create mode 100644 spec/html2rss/web/registry/config_spec.rb create mode 100644 spec/html2rss/web/registry/scrape_policy_spec.rb create mode 100644 spec/html2rss/web/registry/store_spec.rb create mode 100644 spec/html2rss/web/registry/sync_url_resolver_spec.rb create mode 100644 spec/html2rss/web/registry/version_gate_spec.rb diff --git a/Gemfile b/Gemfile index ae3e430ef..8ff642c87 100644 --- a/Gemfile +++ b/Gemfile @@ -4,9 +4,8 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -# gem 'html2rss', path: '../html2rss' -# gem 'html2rss', '~> 0.27' gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' +# Local monorepo override: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'base64' gem 'rack-cache' diff --git a/app/registries/seed/official/.gitkeep b/app/registries/seed/official/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/registries/seed/official/configs/phys.org/weekly.yml b/app/registries/seed/official/configs/phys.org/weekly.yml new file mode 100644 index 000000000..4c0102d03 --- /dev/null +++ b/app/registries/seed/official/configs/phys.org/weekly.yml @@ -0,0 +1,25 @@ +registry: + id: phys.org/weekly +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..83c1c81b6 --- /dev/null +++ b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,34 @@ +registry: + id: support.apple.com/en_gb_ht201222 +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/app/registries/seed/official/manifest.json b/app/registries/seed/official/manifest.json new file mode 100644 index 000000000..d02f057ad --- /dev/null +++ b/app/registries/seed/official/manifest.json @@ -0,0 +1,10 @@ +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test", + "files": { + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + } +} diff --git a/app/web/registry/catalog_change_reporter.rb b/app/web/registry/catalog_change_reporter.rb new file mode 100644 index 000000000..f218090c3 --- /dev/null +++ b/app/web/registry/catalog_change_reporter.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Reports catalog diffs after a registry bundle is promoted to active. + module CatalogChangeReporter + module_function + + ## + # @param registry_id [String] + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [void] + def report!(registry_id:, previous_bundle:, new_bundle:) + diff = build_diff(previous_bundle, new_bundle) + return if diff.empty? + + emit_observability!(registry_id, diff) + emit_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? + end + + ## + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [Hash{Symbol => Object}] + def build_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + previous_version = previous_bundle&.manifest&.version + new_version = new_bundle.manifest.version + previous_ids = catalog_ids(previous_bundle) + new_ids = catalog_ids(new_bundle) + added = new_ids - previous_ids + removed = previous_ids - new_ids + + return {} if previous_version == new_version && added.empty? && removed.empty? + + { + previous_version:, + version: new_version, + added:, + removed: + } + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength + Observability.emit( + event_name: 'registry.catalog_changed', + outcome: 'success', + level: :warn, + details: { + registry_id:, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + } + ) + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_security!(registry_id, diff) + SecurityLogger.log_registry_catalog_changed( + registry_id, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + ) + end + + ## + # @param bundle [Index::RegistryBundle, nil] + # @return [Array] + def catalog_ids(bundle) + return [] unless bundle + + bundle.catalog_entries.map(&:id) + end + + private_class_method :build_diff, :emit_observability!, :emit_security!, :catalog_ids + end + end + end +end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 4fd5b8842..065fcfae5 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -6,6 +6,10 @@ module Html2rss module Web module Registry + ## + # Sync policy parsed from registry YAML. + SyncPolicy = Data.define(:pin_version, :max_version, :auto_promote) + ## # Registry definition parsed from {Config::REGISTRIES_FILE}. Entry = Data.define( @@ -16,7 +20,9 @@ module Registry :sync_url, :catalog, :public_key_id, - :public_key + :public_key, + :sync_policy, + :allowed_channel_domains ) do ## # @return [Hash{String => OpenSSL::PKey::PKey}] @@ -115,14 +121,32 @@ def default_document { precedence: DEFAULT_PRECEDENCE, registries: { - 'official' => { - sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, - catalog: true - } + 'official' => default_official_registry } } end + ## + # @return [Hash{Symbol => Object}] + def default_official_registry + { + sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: default_public_key_pem + } + end + + ## + # @return [String] + def default_public_key_pem + <<~PEM + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- + PEM + end + ## # @param registry_id [String] # @param document [Hash{Symbol => Object}] @@ -141,13 +165,36 @@ def parse_entry(registry_id, document) # @return [Entry] def build_entry(registry_id, raw) sync = sync_section(raw) - mode, resolved_path, sync_channel, sync_url = resolve_mode( - path: raw[:path]&.to_s, - sync_channel: sync[:channel]&.to_s, - sync_url: sync[:url]&.to_s - ) + case raw + in { path: path } if path&.then { |value| !value.empty? } + entry_attributes(registry_id, raw, :path, expand_path(path.to_s), nil, nil) + else + build_sync_entry(registry_id, raw, sync) + end + end - entry_attributes(registry_id, raw, mode, resolved_path, sync_channel, sync_url) + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @param sync [Hash{Symbol => Object}] + # @return [Entry] + def build_sync_entry(registry_id, raw, sync) + sync_channel = sync[:channel]&.to_s + sync_url = sync[:url]&.to_s + resolved_sync_url = resolved_sync_url(sync_url, sync_channel) + entry = entry_attributes(registry_id, raw, :sync, nil, sync_channel, resolved_sync_url) + validate_sync_public_key!(registry_id, entry) + entry + end + + ## + # @param sync_url [String] + # @param sync_channel [String, nil] + # @return [String] + def resolved_sync_url(sync_url, sync_channel) + return sync_url if sync_url && !sync_url.empty? + + SyncUrlResolver.resolve_channel_url(sync_channel) end ## @@ -158,7 +205,8 @@ def build_entry(registry_id, raw) # @param sync_channel [String, nil] # @param sync_url [String, nil] # @return [Entry] - def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists + def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength + sync = sync_section(raw) Entry.new( id: registry_id, mode:, @@ -167,7 +215,9 @@ def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rub sync_url:, catalog: raw.fetch(:catalog, true), public_key_id: public_key_id_for(raw[:public_key_id]&.to_s), - public_key: parse_public_key(raw[:public_key]) + public_key: parse_public_key(raw[:public_key]), + sync_policy: sync_policy_for(raw, sync), + allowed_channel_domains: parse_allowed_channel_domains(raw[:allowed_channel_domains]) ) end @@ -179,34 +229,57 @@ def sync_section(raw) end ## - # @param public_key_id [String, nil] - # @return [String] - def public_key_id_for(public_key_id) - return public_key_id if public_key_id && !public_key_id.empty? + # @param raw [Hash{Symbol => Object}] + # @param sync [Hash{Symbol => Object}] + # @return [SyncPolicy] + def sync_policy_for(raw, sync) + SyncPolicy.new( + pin_version: optional_string(sync[:pin_version]), + max_version: optional_string(sync[:max_version]), + auto_promote: auto_promote?(raw[:auto_promote]) + ) + end - 'html2rss:registry:2026' + ## + # @param value [Object] + # @return [Boolean] + def auto_promote?(value) + value == true end ## - # @param path [String, nil] - # @param sync_channel [String, nil] - # @param sync_url [String, nil] - # @return [Array(Symbol, String, nil, String, nil)] - def resolve_mode(path:, sync_channel:, sync_url:) - return [:path, expand_path(path), nil, nil] if path && !path.empty? + # @param value [Object] + # @return [String, nil] + def optional_string(value) + string = value&.to_s + string.nil? || string.empty? ? nil : string + end - resolved_sync_url = sync_url && !sync_url.empty? ? sync_url : resolve_sync_channel_url(sync_channel) - [:sync, nil, sync_channel, resolved_sync_url] + ## + # @param value [Object] + # @return [Array] + def parse_allowed_channel_domains(value) + Array(value).map { |domain| domain.to_s.strip }.reject(&:empty?) end ## - # @param sync_channel [String, nil] + # @param public_key_id [String, nil] # @return [String] - def resolve_sync_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : DEFAULT_OFFICIAL_SYNC_CHANNEL - return OFFICIAL_RELEASE_URL if channel == DEFAULT_OFFICIAL_SYNC_CHANNEL + def public_key_id_for(public_key_id) + return public_key_id if public_key_id && !public_key_id.empty? + + 'html2rss:registry:2026' + end + + ## + # @param registry_id [String] + # @param entry [Entry] + # @return [void] + def validate_sync_public_key!(registry_id, entry) + return unless entry.public_key.nil? - raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + raise Errors::ConfigError, + "Sync registry '#{registry_id}' requires a pinned public_key in #{registries_file}" end ## diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 6052c4bf4..5c0df5d84 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -50,8 +50,15 @@ def config_for(feed_id) # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows rows = registry_catalog_rows - LocalCatalog.rows.each { |row| rows[row.fetch(:id)] = row } - rows.values.sort_by { |row| row.fetch(:id) } + LocalCatalog.rows.each { |row| rows[row.id] = row } + rows.values.sort_by(&:id).map(&:to_h) + end + + ## + # @param registry_id [String, Symbol] + # @return [RegistryBundle, nil] + def bundle_for(registry_id) + loaded_bundles[registry_id.to_s] end ## @@ -72,7 +79,7 @@ def status private ## - # @return [Hash{String => Hash{Symbol => Object}}] + # @return [Hash{String => RegistryCatalogRow, LocalCatalogRow}] def registry_catalog_rows Config.precedence.each_with_object({}) do |registry_id, rows| next unless Config.catalog_enabled?(registry_id) @@ -81,7 +88,7 @@ def registry_catalog_rows next unless bundle bundle.catalog_entries.each do |entry| - rows[entry.id] ||= wire_registry_row(entry, registry_id) + rows[entry.id] ||= RegistryCatalogRow.from_entry(entry, registry_id) end end end @@ -99,13 +106,15 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength entry = Config.entry(registry_id) directory = bundle_directory(entry) return nil unless directory && File.directory?(directory) + return nil unless active_bundle_present?(directory) bundle = Html2rss::Registry::Bundle.load( directory, - trust: :integrity_only, - public_keys: entry.public_keys + **TrustContext.for_entry(entry, directory).load_options ) - to_registry_bundle(registry_id, bundle) + registry_bundle = to_registry_bundle(registry_id, bundle) + ScrapePolicy.enforce!(entry, registry_bundle) + registry_bundle rescue Html2rss::Registry::Error => error raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end @@ -132,6 +141,13 @@ def bundle_directory(entry) Store.registry_dir(entry.id) end + ## + # @param directory [String] + # @return [Boolean] + def active_bundle_present?(directory) + File.file?(File.join(directory, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + ## # @param feed_id [String, Symbol] # @return [String] @@ -139,26 +155,6 @@ def normalize_feed_id(feed_id) feed_id.to_s.delete_prefix('/').sub(LocalConfig::FEED_EXTENSION_PATTERN, '') end - ## - # @param feed_id [String] - # @return [Hash{Symbol => Object}, nil] - def local_config_for(feed_id) - feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] - return nil unless feed - - deep_dup(feed) - rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound - nil - end - - ## - # @param entry [Html2rss::Registry::CatalogEntry] - # @param registry_id [String] - # @return [Hash{Symbol => Object}] - def wire_registry_row(entry, registry_id) - entry.to_h.merge(source: 'registry', registry: registry_id) - end - ## # @param registry_id [String] # @return [Time, nil] @@ -172,6 +168,18 @@ def bundle_updated_at(registry_id) File.mtime(manifest_path) end + ## + # @param feed_id [String] + # @return [Hash{Symbol => Object}, nil] + def local_config_for(feed_id) + feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] + return nil unless feed + + deep_dup(feed) + rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound + nil + end + ## # @param value [Object] # @return [Object] diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb index 063dd7b29..9623bf45f 100644 --- a/app/web/registry/local_catalog.rb +++ b/app/web/registry/local_catalog.rb @@ -9,7 +9,7 @@ module LocalCatalog module_function ## - # @return [Array Object}>] + # @return [Array] def rows LocalConfig.feeds.filter_map do |feed_name, feed_config| build_row(feed_name, feed_config) @@ -19,7 +19,7 @@ def rows ## # @param feed_name [String, Symbol] # @param feed_config [Hash{Symbol => Object}] - # @return [Hash{Symbol => Object}, nil] + # @return [LocalCatalogRow, nil] def build_row(feed_name, feed_config) directory = feed_config[:directory] || {} title = directory[:title] @@ -33,17 +33,17 @@ def build_row(feed_name, feed_config) # @param directory [Hash{Symbol => Object}] # @param title [String] # @param channel [Hash{Symbol => Object}] - # @return [Hash{Symbol => Object}] + # @return [LocalCatalogRow] def row_identity(feed_name, directory, title, channel) id = feed_name.to_s - { + LocalCatalogRow.new( id:, path: "/#{id}.rss", source: 'local', directory: directory_payload(directory, title), channel: channel_payload(channel, title), parameters: { schema: {}, defaults: {} } - } + ) end ## diff --git a/app/web/registry/local_catalog_row.rb b/app/web/registry/local_catalog_row.rb new file mode 100644 index 000000000..0d95a784f --- /dev/null +++ b/app/web/registry/local_catalog_row.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Catalog wire row for local feeds.yml entries. + LocalCatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source + ) + end + end +end diff --git a/app/web/registry/registry_catalog_row.rb b/app/web/registry/registry_catalog_row.rb new file mode 100644 index 000000000..27d3791e6 --- /dev/null +++ b/app/web/registry/registry_catalog_row.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Catalog wire row for registry-sourced feed configs. + RegistryCatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source, + :registry + ) do + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [RegistryCatalogRow] + def self.from_entry(entry, registry_id) + new( + id: entry.id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + end + end + end +end diff --git a/app/web/registry/scrape_policy.rb b/app/web/registry/scrape_policy.rb new file mode 100644 index 000000000..b341d751f --- /dev/null +++ b/app/web/registry/scrape_policy.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'uri' + +module Html2rss + module Web + module Registry + ## + # Enforces optional channel URL domain allowlists for registry bundles. + module ScrapePolicy + module_function + + ## + # @param entry [Entry] + # @param bundle [Index::RegistryBundle] + # @return [void] + # @raise [Errors::LoadError] when a config channel URL violates the allowlist + def enforce!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + allowed = entry.allowed_channel_domains + return if allowed.nil? || allowed.empty? + + bundle.configs.each do |feed_id, config| + channel_url = config.dig(:channel, :url) + host = host_for(channel_url) + next if host && allowed.any? { |domain| domain_allowed?(host, domain) } + + raise Errors::LoadError, + "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ + "'#{host || channel_url}' is not allowed by allowed_channel_domains" + end + end + + ## + # @param host [String] + # @param allowed_domain [String] + # @return [Boolean] + def domain_allowed?(host, allowed_domain) + normalized_host = host.downcase + normalized_domain = allowed_domain.downcase + normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") + end + + ## + # @param channel_url [String, nil] + # @return [String, nil] + def host_for(channel_url) + return nil if channel_url.to_s.strip.empty? + + URI.parse(channel_url).host + rescue URI::InvalidURIError + nil + end + + private_class_method :host_for + end + end + end +end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index fc9e7459e..1bd69f28d 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -8,12 +8,14 @@ module Web module Registry ## # Manages on-disk registry bundle directories under {data_root}. - module Store + module Store # rubocop:disable Metrics/ModuleLength DEFAULT_DATA_ROOT = 'tmp/registry-data' DEFAULT_SEED_ROOT = '/app/registries/seed' SYNC_STATE_FILE = '.sync-state.json' - class << self + SyncState = Data.define(:last_error, :last_sync_at) + + class << self # rubocop:disable Metrics/ClassLength ## # @return [String] root directory for extracted registry bundles def data_root @@ -47,6 +49,66 @@ def bundle_present?(registry_id) bundle_present_at?(registry_dir(registry_id)) end + ## + # @param registry_id [String, Symbol] + # @return [String] verified staging directory for a registry id + def staging_dir(registry_id) + File.join(registry_dir(registry_id), '.staging') + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def staged_present?(registry_id) + bundle_present_at?(staging_dir(registry_id)) + end + + ## + # @param registry_id [String, Symbol] + # @return [String, nil] + def staged_version(registry_id) + manifest_version_at(staging_dir(registry_id)) + end + + ## + # Writes a verified bundle to the registry staging directory. + # + # @param registry_id [String, Symbol] + # @param staged_dir [String] verified bundle directory to stage + # @return [String] staging directory + def stage_bundle!(registry_id, staged_dir) + raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) + + target = staging_dir(registry_id) + parent = registry_dir(registry_id) + FileUtils.mkdir_p(parent) + + backup = "#{target}.backup.#{Process.pid}" + promote_bundle!(staged_dir, target, backup) + target + end + + ## + # Promotes a verified staging bundle to the active registry directory. + # + # @param registry_id [String, Symbol] + # @return [String] active bundle directory + def promote_staged!(registry_id) # rubocop:disable Metrics/MethodLength + staged = staging_dir(registry_id) + raise Errors::LoadError, "No staged bundle for '#{registry_id}'" unless staged_present?(registry_id) + + temp_root = Dir.mktmpdir('registry-promote-') + temp_staged = File.join(temp_root, 'bundle') + FileUtils.mv(staged, temp_staged) + + active = registry_dir(registry_id) + backup = "#{active}.backup.#{Process.pid}" + promote_bundle!(temp_staged, active, backup) + active + ensure + FileUtils.rm_rf(temp_root) if temp_root + end + ## # Atomically replaces the active bundle directory for a registry id. # @@ -84,9 +146,13 @@ def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMe ## # @param registry_id [String, Symbol] - # @return [Hash{Symbol => Object}] + # @return [SyncState] def sync_state(registry_id) - read_sync_state.fetch(registry_id.to_s, {}) + raw = read_sync_state.fetch(registry_id.to_s, {}) + SyncState.new( + last_error: raw['last_error'], + last_sync_at: parse_sync_time(raw['last_sync_at']) + ) end ## @@ -143,7 +209,30 @@ def promote_bundle!(staged_dir, target, backup) # @param path [String] # @return [Boolean] def bundle_present_at?(path) - File.directory?(path) && !Dir.empty?(path) + File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + + ## + # @param path [String] + # @return [String, nil] + def manifest_version_at(path) + return nil unless bundle_present_at?(path) + + manifest = Html2rss::Registry::Manifest.parse( + File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + manifest.version + rescue Html2rss::Registry::ManifestError + nil + end + + ## + # @param value [String, nil] + # @return [Time, nil] + def parse_sync_time(value) + return nil if value.to_s.empty? + + Time.parse(value) end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 4d7f30cf9..f96c3448e 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true require 'fileutils' -require 'net/http' -require 'json' -require 'stringio' -require 'uri' module Html2rss module Web @@ -12,22 +8,15 @@ module Registry ## # Fetches, verifies, and stores signed registry bundles. module Sync # rubocop:disable Metrics/ModuleLength - OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL - OFFICIAL_GITHUB_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' - OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' - - DEFAULT_ALLOWED_HOSTS = %w[ - api.github.com - github.com - objects.githubusercontent.com - ].freeze - FETCH_OPEN_TIMEOUT_SECONDS = 10 - FETCH_READ_TIMEOUT_SECONDS = 60 - MAX_RESPONSE_BYTES = Html2rss::Registry::Archive::MAX_TARBALL_BYTES - BACKGROUND_JITTER_FRACTION = 0.1 - - SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) + SyncStatus = Data.define( + :registry_id, + :mode, + :version, + :staged_version, + :updated_at, + :sync_url, + :last_error + ) @boot_mutex = Mutex.new @boot_started = false @@ -43,7 +32,7 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - resolve_download_url(entry) + SyncUrlResolver.resolve(entry) end ## @@ -52,30 +41,21 @@ def sync_url_for(registry_id) # @param registry_id [String, Symbol] # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] - def run(registry_id:, dry_run: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - entry = Config.entry(registry_id) - if entry.mode == :path - raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" - end + def run(registry_id:, dry_run: false) + SyncCoordinator.run(registry_id) { run!(registry_id:, dry_run:) } + end - staging_root = nil - download_url = resolve_download_url(entry) - staging_root, staged_dir = fetch_and_verify!(entry, download_url) - unless dry_run - Store.swap!(registry_id, staged_dir) - Index.reload! - record_success!(registry_id) - end - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) - rescue StandardError => error - record_failure!(registry_id, error) unless dry_run - raise - ensure - FileUtils.rm_rf(staging_root) if staging_root + ## + # Promotes a verified staging bundle to the active registry directory. + # + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def promote_staged!(registry_id:) + SyncCoordinator.run(registry_id) { promote_staged_bundle!(registry_id:) } end ## - # @param registry_id [String, Symbol, nil] + # @param registry_id [String, Symbol] # @return [Array] def status(registry_id: nil) rows = Index.current.status @@ -149,6 +129,90 @@ def unusable_sync_registries private + ## + # @param registry_id [String, Symbol] + # @param dry_run [Boolean] + # @return [SyncStatus] + def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" + end + + staging_root = nil + download_url = SyncUrlResolver.resolve(entry) + staging_root, staged_dir = fetch_and_verify!(entry, download_url) + unless dry_run + if entry.sync_policy.auto_promote + activate_bundle!(registry_id, staged_dir) + else + Store.stage_bundle!(registry_id, staged_dir) + record_success!(registry_id, promoted: false) + end + end + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + rescue StandardError => error + record_failure!(registry_id, error) unless dry_run + raise + ensure + FileUtils.rm_rf(staging_root) if staging_root + end + + ## + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def promote_staged_bundle!(registry_id:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; promote is not applicable" + end + unless Store.staged_present?(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' has no staged bundle" + end + + previous_bundle = Index.current.bundle_for(registry_id) + Store.promote_staged!(registry_id) + Index.reload! + report_catalog_change!(registry_id, previous_bundle) + record_success!(registry_id, promoted: true) + Observability.emit( + event_name: 'registry.promote_staged', + outcome: 'success', + details: { + registry_id: registry_id.to_s, + version: Index.current.status.find { |row| row.id == registry_id.to_s }&.version + } + ) + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + end + + ## + # @param registry_id [String, Symbol] + # @param staged_dir [String] + # @return [void] + def activate_bundle!(registry_id, staged_dir) + previous_bundle = Index.current.bundle_for(registry_id) + Store.swap!(registry_id, staged_dir) + Index.reload! + report_catalog_change!(registry_id, previous_bundle) + record_success!(registry_id, promoted: true) + end + + ## + # @param registry_id [String, Symbol] + # @param previous_bundle [Index::RegistryBundle, nil] + # @return [void] + def report_catalog_change!(registry_id, previous_bundle) + new_bundle = Index.current.bundle_for(registry_id) + return unless new_bundle + + CatalogChangeReporter.report!( + registry_id: registry_id.to_s, + previous_bundle:, + new_bundle: + ) + end + ## # @return [Boolean] def skip_boot? @@ -213,41 +277,22 @@ def background_jitter_seconds(interval_hours) rand(max_jitter) end - ## - # @param entry [Entry] - # @return [String] - def resolve_download_url(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? - return resolve_official_download_url if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - - raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" - end - - ## - # @return [String] - def resolve_official_download_url - response_body = Fetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end + BACKGROUND_JITTER_FRACTION = 0.1 + private_constant :BACKGROUND_JITTER_FRACTION ## # @param entry [Entry] # @param download_url [String] # @return [Array(String, String)] staging root and verified bundle directory def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tarball = Fetcher.fetch!(download_url) + tarball = SyncFetcher.fetch!(download_url) staging_root = Dir.mktmpdir('registry-sync-') staged_dir = File.join(staging_root, 'bundle') + tarball_path = File.join(staging_root, 'download.tar.gz') FileUtils.mkdir_p(staged_dir) + File.binwrite(tarball_path, tarball) - StringIO.open(tarball) do |io| + File.open(tarball_path, 'rb') do |io| Html2rss::Registry::Archive.extract!(io, into: staged_dir) end @@ -256,6 +301,7 @@ def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Me trust: :signed, public_keys: entry.public_keys ) + enforce_max_version!(entry, staged_dir) [staging_root, staged_dir] rescue Html2rss::Registry::VerificationError => error log_signature_failure!(entry.id, error.message) if signature_failure?(error) @@ -279,16 +325,46 @@ def log_signature_failure!(registry_id, message) SecurityLogger.log_registry_signature_failure(registry_id, message) end + ## + # @param entry [Entry] + # @param staged_dir [String] + # @return [void] + def enforce_max_version!(entry, staged_dir) + manifest = read_manifest!(staged_dir) + max_version = entry.sync_policy.max_version + return unless max_version + + return unless VersionGate.exceeds_max?(manifest.version, max_version) + + raise Errors::SyncError, + "Registry '#{entry.id}' manifest version '#{manifest.version}' exceeds max_version '#{max_version}'" + end + + ## + # @param staged_dir [String] + # @return [Html2rss::Registry::Manifest] + def read_manifest!(staged_dir) + Html2rss::Registry::Manifest.parse( + File.read(File.join(staged_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + end + ## # @param registry_id [String] + # @param promoted [Boolean] # @return [void] - def record_success!(registry_id) + def record_success!(registry_id, promoted:) # rubocop:disable Metrics/MethodLength Store.write_sync_state!(registry_id, last_error: nil) row = Index.current.status.find { |entry| entry.id == registry_id.to_s } Observability.emit( event_name: 'registry.sync', outcome: 'success', - details: { registry_id:, version: row&.version } + details: { + registry_id:, + version: row&.version, + staged_version: Store.staged_version(registry_id), + promoted: + } ) end @@ -309,118 +385,20 @@ def record_failure!(registry_id, error) ## # @param row [Index::StatusEntry] # @return [SyncStatus] - def sync_status_for(row) + def sync_status_for(row) # rubocop:disable Metrics/MethodLength entry = Config.entry(row.id) state = Store.sync_state(row.id) SyncStatus.new( registry_id: row.id, mode: row.sync_mode, version: row.version, + staged_version: Store.staged_version(row.id), updated_at: row.updated_at, - sync_url: entry.mode == :sync ? resolve_download_url(entry) : nil, - last_error: state['last_error'] + sync_url: entry.mode == :sync ? SyncUrlResolver.resolve(entry) : nil, + last_error: state.last_error ) end end # rubocop:enable Metrics/ClassLength - - ## - # HTTPS fetcher with host allowlisting and redirect rejection. - module Fetcher - module_function - - ## - # @param url [String] - # @return [String] response body - def fetch!(url) - uri = parse_https_uri!(url) - ensure_allowed_host!(uri.host) - perform_request!(uri) - end - - ## - # @param url [String] - # @return [URI::HTTPS] - def parse_https_uri!(url) - uri = URI(url) - unless uri.is_a?(URI::HTTPS) - raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" - end - - uri - end - - ## - # @param host [String] - # @return [void] - def ensure_allowed_host!(host) - allowed = DEFAULT_ALLOWED_HOSTS + extra_allowed_hosts - return if allowed.include?(host) - - raise Errors::SyncError, "Registry sync host not allowed: #{host}" - end - - ## - # @return [Array] - def extra_allowed_hosts - ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - .split(',') - .map(&:strip) - .reject(&:empty?) - end - - ## - # @param uri [URI::HTTPS] - # @return [String] - def perform_request!(uri) # rubocop:disable Metrics/MethodLength - response = nil - Net::HTTP.start( - uri.host, - uri.port, - use_ssl: true, - open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, - read_timeout: FETCH_READ_TIMEOUT_SECONDS - ) do |http| - request = Net::HTTP::Get.new(uri) - request['Accept'] = 'application/octet-stream' - request['User-Agent'] = 'html2rss-web/registry-sync' - response = http.request(request) - end - - reject_redirect!(response) - reject_error_status!(response) - read_body!(response) - end - - ## - # @param response [Net::HTTPResponse] - # @return [void] - def reject_redirect!(response) - return unless response.is_a?(Net::HTTPRedirection) - - raise Errors::SyncError, 'Registry sync rejects HTTP redirects' - end - - ## - # @param response [Net::HTTPResponse] - # @return [void] - def reject_error_status!(response) - return if response.is_a?(Net::HTTPSuccess) - - raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" - end - - ## - # @param response [Net::HTTPResponse] - # @return [String] - def read_body!(response) - body = response.body.to_s - if body.bytesize > MAX_RESPONSE_BYTES - raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" - end - - body - end - end end # rubocop:enable Metrics/ModuleLength end end diff --git a/app/web/registry/sync_coordinator.rb b/app/web/registry/sync_coordinator.rb new file mode 100644 index 000000000..2bd641e07 --- /dev/null +++ b/app/web/registry/sync_coordinator.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Serializes sync work per registry id to avoid concurrent bundle swaps. + module SyncCoordinator + REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } + + module_function + + ## + # @param registry_id [String, Symbol] + # @yield runs sync work exclusively for the registry id + # @return [Object] block result + def run(registry_id, &) + REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) + end + end + end + end +end diff --git a/app/web/registry/sync_fetcher.rb b/app/web/registry/sync_fetcher.rb new file mode 100644 index 000000000..6d13712a3 --- /dev/null +++ b/app/web/registry/sync_fetcher.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module Html2rss + module Web + module Registry + ## + # HTTPS fetcher with host allowlisting and bounded redirect following. + module SyncFetcher + DEFAULT_ALLOWED_HOSTS = %w[ + api.github.com + github.com + objects.githubusercontent.com + release-assets.githubusercontent.com + ].freeze + FETCH_OPEN_TIMEOUT_SECONDS = 10 + FETCH_READ_TIMEOUT_SECONDS = 60 + MAX_RESPONSE_BYTES = 52_428_800 + DEFAULT_MAX_REDIRECTS = 5 + + RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do + ## + # @return [RedirectPolicy] + def self.default + new(max_hops: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) + end + + ## + # @return [Array] + def self.default_allowed_hosts + DEFAULT_ALLOWED_HOSTS + SyncFetcher.extra_allowed_hosts + end + end + + module_function + + ## + # @param url [String] + # @param policy [RedirectPolicy] + # @return [String] response body + def fetch!(url, policy: RedirectPolicy.default) + uri = parse_https_uri!(url) + follow_redirects!(uri, policy) + end + + ## + # @param uri [URI::HTTPS] + # @param policy [RedirectPolicy] + # @return [String] + def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + hops = 0 + + loop do + ensure_allowed_host!(uri.host, policy.allowed_hosts) + response = perform_request!(uri) + + case response + in Net::HTTPRedirection + hops += 1 + raise Errors::SyncError, 'Registry sync exceeded redirect limit' if hops > policy.max_hops + + location = response['location'] + raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? + + uri = parse_https_uri!(URI.join(uri, location).to_s) + next + else + reject_error_status!(response) + return read_body!(response) + end + end + end + + ## + # @return [Array] + def extra_allowed_hosts + ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + .split(',') + .map(&:strip) + .reject(&:empty?) + end + + ## + # @param url [String] + # @return [URI::HTTPS] + def parse_https_uri!(url) + uri = URI(url) + unless uri.is_a?(URI::HTTPS) + raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" + end + + uri + end + + ## + # @param host [String] + # @param allowed_hosts [Array] + # @return [void] + def ensure_allowed_host!(host, allowed_hosts) + return if allowed_hosts.include?(host) + + raise Errors::SyncError, "Registry sync host not allowed: #{host}" + end + + ## + # @param uri [URI::HTTPS] + # @return [Net::HTTPResponse] + def perform_request!(uri) # rubocop:disable Metrics/MethodLength + response = nil + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: true, + open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, + read_timeout: FETCH_READ_TIMEOUT_SECONDS + ) do |http| + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/octet-stream' + request['User-Agent'] = 'html2rss-web/registry-sync' + response = http.request(request) + end + response + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_error_status!(response) + return if response.is_a?(Net::HTTPSuccess) + + raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" + end + + ## + # @param response [Net::HTTPResponse] + # @return [String] + def read_body!(response) + body = response.body.to_s + if body.bytesize > MAX_RESPONSE_BYTES + raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" + end + + body + end + end + end + end +end diff --git a/app/web/registry/sync_url_resolver.rb b/app/web/registry/sync_url_resolver.rb new file mode 100644 index 000000000..d035c7396 --- /dev/null +++ b/app/web/registry/sync_url_resolver.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'json' + +module Html2rss + module Web + module Registry + ## + # Resolves registry sync download URLs from config entries and official release metadata. + module SyncUrlResolver + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_GITHUB_TAG_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + + module_function + + ## + # @param entry [Entry] + # @return [String] + def resolve(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + + pin_version = entry.sync_policy.pin_version + if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + + return resolve_official_download_url + end + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param tag [String] + # @return [String] + def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + response_body = SyncFetcher.fetch!(api_url) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + unless url + raise Errors::SyncError, + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + end + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = SyncFetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + end + end + end +end diff --git a/app/web/registry/trust_context.rb b/app/web/registry/trust_context.rb new file mode 100644 index 000000000..960593a10 --- /dev/null +++ b/app/web/registry/trust_context.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Trust mode and verification keys passed to {Html2rss::Registry::Bundle.load}. + TrustContext = Data.define(:mode, :public_keys) do + ## + # @param entry [Entry] + # @param bundle_dir [String] + # @return [TrustContext] + def self.for_entry(entry, bundle_dir) + case entry.mode + in :path + integrity_only + in :sync + signed_bundle?(bundle_dir, entry) ? signed(entry.public_keys) : integrity_only + end + end + + ## + # @return [TrustContext] + def self.integrity_only + new(mode: :integrity_only, public_keys: {}) + end + + ## + # @param public_keys [Hash{String => OpenSSL::PKey::PKey}] + # @return [TrustContext] + def self.signed(public_keys) + new(mode: :signed, public_keys:) + end + + ## + # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} + def load_options + { trust: mode, public_keys: } + end + + ## + # @param bundle_dir [String] + # @param entry [Entry] + # @return [Boolean] + def self.signed_bundle?(bundle_dir, entry) + signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) + File.file?(signature_path) && !entry.public_keys.empty? + end + + private_class_method :signed_bundle? + end + end + end +end diff --git a/app/web/registry/version_gate.rb b/app/web/registry/version_gate.rb new file mode 100644 index 000000000..27f6aadbd --- /dev/null +++ b/app/web/registry/version_gate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Compares registry manifest versions for sync policy caps. + module VersionGate + module_function + + ## + # @param manifest_version [String] + # @param max_version [String, nil] + # @return [Boolean] true when manifest_version is newer than max_version + def exceeds_max?(manifest_version, max_version) + return false if max_version.nil? || max_version.empty? + + compare(normalize(manifest_version), normalize(max_version)).positive? + end + + ## + # @param version [String] + # @return [String] + def normalize(version) + version.to_s.delete_prefix('v') + end + + ## + # @param left [String] + # @param right [String] + # @return [Integer] + def compare(left, right) + Gem::Version.new(left) <=> Gem::Version.new(right) + rescue ArgumentError + left <=> right + end + end + end + end +end diff --git a/app/web/security/security_logger.rb b/app/web/security/security_logger.rb index e6293584b..894345e81 100644 --- a/app/web/security/security_logger.rb +++ b/app/web/security/security_logger.rb @@ -76,6 +76,13 @@ def log_registry_signature_failure(registry_id, reason) log_event('registry_signature_failure', { registry_id:, reason: }, severity: :warn) end + # @param registry_id [String] + # @param details [Hash{Symbol => Object}] + # @return [void] + def log_registry_catalog_changed(registry_id, details) + log_event('registry_catalog_changed', { registry_id:, **details }, severity: :warn) + end + private # @param event_type [String] diff --git a/bin/registry-sync b/bin/registry-sync index 1c013a099..e8321a8b2 100755 --- a/bin/registry-sync +++ b/bin/registry-sync @@ -9,12 +9,13 @@ module RegistrySyncCLI module_function def run(argv = ARGV) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - options = { status: false, registry_id: nil, dry_run: false } + options = { status: false, registry_id: nil, dry_run: false, promote: false } OptionParser.new do |opts| opts.banner = 'Usage: bin/registry-sync [options]' opts.on('--status', 'Print registry sync status and exit') { options[:status] = true } opts.on('--registry ID', 'Sync a single registry id') { |value| options[:registry_id] = value } opts.on('--dry-run', 'Fetch and verify without swapping the active bundle') { options[:dry_run] = true } + opts.on('--promote', 'Promote a verified staging bundle to active') { options[:promote] = true } end.parse!(argv) if options[:status] @@ -22,6 +23,13 @@ module RegistrySyncCLI exit Html2rss::Web::Registry::Sync.cli_exit_code end + if options[:promote] + target_registry_ids(options[:registry_id]).each do |registry_id| + Html2rss::Web::Registry::Sync.promote_staged!(registry_id:) + end + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + target_registry_ids(options[:registry_id]).each do |registry_id| Html2rss::Web::Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) end @@ -36,13 +44,14 @@ module RegistrySyncCLI def print_status_table(registry_id) # rubocop:disable Metrics/MethodLength rows = Html2rss::Web::Registry::Sync.status(registry_id:) - headers = %w[registry mode version updated_at sync_url last_error] + headers = %w[registry mode version staged_version updated_at sync_url last_error] puts headers.join("\t") rows.each do |row| puts [ row.registry_id, row.mode, row.version || '-', + row.staged_version || '-', format_time(row.updated_at), row.sync_url || '-', row.last_error || '-' diff --git a/config/registries.yml b/config/registries.yml index 3ebeca8e5..9c3d4af85 100644 --- a/config/registries.yml +++ b/config/registries.yml @@ -7,5 +7,16 @@ registries: channel: html2rss-official catalog: true public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- # Pin the publisher Ed25519 public key for signed network sync. # Seed bundles copied from the image use integrity-only verification. + # Production: keep auto_promote false and promote manually after review. + # auto_promote: false + # sync: + # pin_version: v2026.08.22 + # max_version: v2026.08.22 + # allowed_channel_domains: + # - anthropic.com diff --git a/docs/README.md b/docs/README.md index 0fcf1b7f2..7c27ebb33 100644 --- a/docs/README.md +++ b/docs/README.md @@ -298,15 +298,20 @@ Inside the Dev Container: bin/registry-sync --status ``` -Columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any registry reports `last_error`. +Columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any sync-mode registry lacks a usable on-disk bundle. -Sync a single registry or dry-run without swapping the active bundle: +Sync, dry-run, or promote a staged bundle: ```bash bin/registry-sync --registry official bin/registry-sync --registry official --dry-run +bin/registry-sync --promote --registry official ``` +Production recommendation: keep `auto_promote: false` (default), pin `sync.pin_version` to the approved configs tag, run sync to stage a verified bundle, then promote manually after review. Use `sync.max_version` as an incident freeze cap and set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh. + +Optional hardening: `allowed_channel_domains` suffix-matches every registry config `channel.url` host at bundle load time. + ### Boot behavior `Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): @@ -328,6 +333,9 @@ registries: official: sync: channel: html2rss-official + pin_version: v2026.08.22 # optional + max_version: v2026.08.22 # optional incident freeze + auto_promote: false # default; verified bundles stage until --promote catalog: true public_key_id: html2rss:registry:2026 public_key: | @@ -348,6 +356,9 @@ registries: - **`precedence`** — merge order for feed lookup; first match wins. - **`sync.url`** — direct tarball URL, or use `sync.channel: html2rss-official` for the default GitHub release asset. +- **`sync.pin_version` / `sync.max_version`** — fetch a specific tag or reject manifests newer than the cap. +- **`auto_promote: false`** — default; verified bundles land in `REGISTRY_DATA_ROOT//.staging/` until `bin/registry-sync --promote`. +- **`allowed_channel_domains`** — optional suffix allowlist enforced when the bundle loads. - **`catalog: false`** — private registry: configs are served at `/{registry.id}.rss` but excluded from the public catalog API (privacy for internal feeds). - Restrict outbound hosts with `REGISTRY_SYNC_ALLOWED_HOSTS` (comma-separated hostnames). @@ -368,7 +379,7 @@ The directory must contain `manifest.json`, optional `manifest.sig`, and `config 1. Publish a new bundle signed with the new key (`public_key_id` in `manifest.json`). 2. Update `public_key_id` and `public_key` in `registries.yml` on every instance **before** or together with the first release that requires the new key. -3. Re-sync (`bin/registry-sync` or wait for background refresh). Failed verification leaves the previous bundle active and records `last_error`. +3. Re-sync (`bin/registry-sync`) and promote when `auto_promote: false` (`bin/registry-sync --promote`). Failed verification leaves the previous bundle active and records `last_error`. --- diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index c79a907d4..eebc337a7 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -37,12 +37,14 @@ Source of truth for feed YAML and signed release artifacts. ``` 3. Tag the release (tag name becomes `manifest.json` `version` in CI via `REGISTRY_VERSION: ${{ github.ref_name }}`): ```bash - git tag 2026.08.22 # example; use your release tag - git push origin 2026.08.22 + git tag v2026.08.22 # example; use your release tag + git push origin v2026.08.22 ``` 4. Tag push triggers [`.github/workflows/release.yml`](../../html2rss-configs/.github/workflows/release.yml): - `make registry-build -- --sign` - - uploads `dist/registry-bundle.tar.gz` to the GitHub Release + - uploads `dist/registry-bundle.tar.gz` to a **draft** GitHub Release (human publishes after review) + +**Draft → publish:** CI creates a draft release. Maintainers review the asset, release notes, and tag diff, then click **Publish release** in GitHub. Draft releases are invisible to `/releases/latest` until published. ### 1.3 `html2rss-web` (runtime + Docker image) @@ -71,17 +73,27 @@ openssl pkey -in registry-signing.pem -pubout -out registry-signing.pub Keep the private key offline except where signing happens. -### 2.2 Configure `html2rss-configs` CI secret +### 2.2 Configure `html2rss-configs` CI secrets -1. In the `html2rss-configs` GitHub repo, add secret **`REGISTRY_SIGNING_KEY`** with the full PEM contents of `registry-signing.pem`. -2. Release CI reads it here: +In the `html2rss-configs` GitHub repo, add secrets on the **`registry-release`** environment: - ```yaml - env: - REGISTRY_VERSION: ${{ github.ref_name }} - REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} - run: make registry-build -- --sign - ``` +| Secret | Purpose | +| --- | --- | +| `REGISTRY_SIGNING_KEY` | Full PEM contents of `registry-signing.pem` (private key; used by `make registry-build -- --sign`) | +| `REGISTRY_PUBLIC_KEY_PEM` | Full PEM contents of `registry-signing.pub` (public key; used by the release verify step — not a repo fixture) | + +Release CI reads them here: + +```yaml +env: + REGISTRY_VERSION: ${{ github.ref_name }} + REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} +run: make registry-build -- --sign +``` + +The verify step loads the public key from `REGISTRY_PUBLIC_KEY_PEM` in-process (`OpenSSL::PKey.read(ENV.fetch('REGISTRY_PUBLIC_KEY_PEM'))`); no committed PEM file in the configs repo. + +At go-live, the same public key must appear in `html2rss-web/config/registries.yml` (`public_key` / `public_key_id`) so runtime sync verification matches CI. ### 2.3 Pin the public key in `html2rss-web` @@ -92,16 +104,26 @@ registries: official: sync: channel: html2rss-official + pin_version: v2026.08.22 # optional: fetch this tag only + max_version: v2026.08.22 # optional: reject newer manifests (incident freeze) + auto_promote: false # default false; verified bundles stage until manual promote catalog: true public_key_id: html2rss:registry:2026 public_key: | -----BEGIN PUBLIC KEY----- ... -----END PUBLIC KEY----- + allowed_channel_domains: # optional suffix allowlist for channel.url domains + - anthropic.com + - github.com ``` - `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). - Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). +- **`auto_promote: false`** (default) writes verified bundles to `REGISTRY_DATA_ROOT//.staging/` without changing the active catalog. Promote after review with `bin/registry-sync --promote --registry official`. +- **`pin_version`** resolves the GitHub tag release API instead of `/releases/latest`. +- **`max_version`** rejects sync when the verified manifest version is newer than the cap (incident freeze). +- **`allowed_channel_domains`** rejects bundle load when any config `channel.url` host is outside the suffix allowlist. ### 2.4 Tag and publish the configs release @@ -311,16 +333,17 @@ Inside the running container (or Dev Container): bin/registry-sync --status ``` -Tab-separated columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. +Tab-separated columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. - Exit code **0** — all sync-mode registries have a usable on-disk bundle. - Exit code **1** — at least one sync registry lacks a bundle (see `Registry::Sync.unusable_sync_registries`). -Single registry or dry-run: +Single registry, dry-run, or promote staged bundle: ```bash bin/registry-sync --registry official bin/registry-sync --registry official --dry-run +bin/registry-sync --promote --registry official ``` ### 6.2 Instance metadata API @@ -360,15 +383,34 @@ Expect HTTP **200** and valid RSS XML. Configs releases are independent of web image releases. To refresh feeds on a running instance: -### 7.1 Manual sync +### 7.1 Manual sync and promote + +Production recommendation: keep `auto_promote: false`, set `sync.pin_version` to the approved tag, fetch with sync, then promote manually after review. ```bash bin/registry-sync --registry official +bin/registry-sync --status # staged_version shows verified bundle +bin/registry-sync --promote --registry official +``` + +Fetches the pinned or latest signed tarball, verifies signature + digests, and either stages (`auto_promote: false`) or atomically swaps the active bundle (`auto_promote: true`). + +### 7.2 Incident freeze + +To block uptake of a newer configs release without disabling sync entirely: + +```yaml +registries: + official: + sync: + channel: html2rss-official + max_version: v2026.08.21 + auto_promote: false ``` -Fetches the latest signed tarball, verifies signature + digests, atomically swaps the active bundle, and reloads `Registry::Index`. +Set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh while investigating. -### 7.2 Automatic refresh +### 7.3 Automatic refresh | Mechanism | Env var | Default | Behavior | | --- | --- | --- | --- | diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 2939a0d7b..395c56397 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -13,7 +13,7 @@ export type GetApiMetadataData = { export type GetApiMetadataResponses = { /** - * returns catalog pointer metadata + * returns registry status metadata */ 200: { data: { @@ -31,6 +31,12 @@ export type GetApiMetadataResponses = { access_token_required: boolean; enabled: boolean; }; + registries: Array<{ + id: string; + sync_mode: string; + updated_at: string; + version: string; + }>; }; }; success: boolean; @@ -77,39 +83,14 @@ export type GetConfigCatalogResponses = { id: string; parameters: { defaults: { - blog?: string | null; - id?: string | null; - region?: string | null; - repository?: string | null; - section?: string | null; - user_id?: string | null; - username?: string | null; + [key: string]: unknown; }; schema: { - blog?: { - type: string; - } | null; - id?: { - type: string; - } | null; - region?: { - type: string; - } | null; - repository?: { - type: string; - } | null; - section?: { - type: string; - } | null; - user_id?: { - type: string; - } | null; - username?: { - type: string; - } | null; + [key: string]: unknown; }; }; path: string; + registry?: string; source: string; }>; }; diff --git a/public/openapi.yaml b/public/openapi.yaml index 22991265b..d112abdda 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -173,7 +173,6 @@ paths: - channel - parameters - source - - registry type: object type: array required: diff --git a/spec/fixtures/registries/keys/test-key.pem b/spec/fixtures/registries/keys/test-key.pem new file mode 100644 index 000000000..d1583ef45 --- /dev/null +++ b/spec/fixtures/registries/keys/test-key.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIKWA7CdQvmMCa06H6jojjHE30xRV9Ps823kjgqWKvfSE +-----END PRIVATE KEY----- diff --git a/spec/fixtures/registries/keys/test-key.pub b/spec/fixtures/registries/keys/test-key.pub new file mode 100644 index 000000000..236aab386 --- /dev/null +++ b/spec/fixtures/registries/keys/test-key.pub @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= +-----END PUBLIC KEY----- diff --git a/spec/fixtures/registries/official/configs/phys.org/weekly.yml b/spec/fixtures/registries/official/configs/phys.org/weekly.yml index 763f70f2b..4c0102d03 100644 --- a/spec/fixtures/registries/official/configs/phys.org/weekly.yml +++ b/spec/fixtures/registries/official/configs/phys.org/weekly.yml @@ -1,3 +1,5 @@ +registry: + id: phys.org/weekly directory: topics: - science diff --git a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml index 76afeafbe..83c1c81b6 100644 --- a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml +++ b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml @@ -1,3 +1,5 @@ +registry: + id: support.apple.com/en_gb_ht201222 directory: topics: - tech diff --git a/spec/fixtures/registries/official/manifest.json b/spec/fixtures/registries/official/manifest.json index 17d99eece..d02f057ad 100644 --- a/spec/fixtures/registries/official/manifest.json +++ b/spec/fixtures/registries/official/manifest.json @@ -4,7 +4,7 @@ "version": "test-fixture", "public_key_id": "test", "files": { - "configs/phys.org/weekly.yml": "0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373", - "configs/support.apple.com/en_gb_ht201222.yml": "3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac" + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" } } diff --git a/spec/fixtures/registries/private/configs/secret.example/private.yml b/spec/fixtures/registries/private/configs/secret.example/private.yml index 6b516f6c2..14a119e7f 100644 --- a/spec/fixtures/registries/private/configs/secret.example/private.yml +++ b/spec/fixtures/registries/private/configs/secret.example/private.yml @@ -1,3 +1,5 @@ +registry: + id: secret.example/private directory: title: "Private Corp Feed" summary: "Hidden from catalog." diff --git a/spec/fixtures/registries/private/manifest.json b/spec/fixtures/registries/private/manifest.json index 0aacc3ef5..6d901dbbf 100644 --- a/spec/fixtures/registries/private/manifest.json +++ b/spec/fixtures/registries/private/manifest.json @@ -1,9 +1,9 @@ { "format": "registry.v1", "registry_id": "private", - "version": "test-private", + "version": "test-fixture", "public_key_id": "test", "files": { - "configs/secret.example/private.yml": "196760640262a019c0b32e08d2f5272c08b11b10b5880dc13266922a30c872a3" + "configs/secret.example/private.yml": "f3b4f6d567e44960be89c171a915a5451266998090ca9ffdb394050bcbbd0dd2" } } diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml index 763f70f2b..4c0102d03 100644 --- a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml +++ b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml @@ -1,3 +1,5 @@ +registry: + id: phys.org/weekly directory: topics: - science diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml index 76afeafbe..83c1c81b6 100644 --- a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml +++ b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml @@ -1,3 +1,5 @@ +registry: + id: support.apple.com/en_gb_ht201222 directory: topics: - tech diff --git a/spec/fixtures/registries/sync/bundle/manifest.json b/spec/fixtures/registries/sync/bundle/manifest.json index e74570c9a..4e20cdcbf 100644 --- a/spec/fixtures/registries/sync/bundle/manifest.json +++ b/spec/fixtures/registries/sync/bundle/manifest.json @@ -1 +1,10 @@ -{"format":"registry.v1","registry_id":"official","version":"test-fixture","public_key_id":"test-key","files":{"configs/phys.org/weekly.yml":"0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373","configs/support.apple.com/en_gb_ht201222.yml":"3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac"}} \ No newline at end of file +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test-key", + "files": { + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + } +} diff --git a/spec/fixtures/registries/sync/registries.yml b/spec/fixtures/registries/sync/registries.yml index e307db29a..4f1ab3dff 100644 --- a/spec/fixtures/registries/sync/registries.yml +++ b/spec/fixtures/registries/sync/registries.yml @@ -5,6 +5,7 @@ registries: official: sync: url: https://registry.test.example/registry-bundle.tar.gz + auto_promote: true catalog: true public_key_id: test-key public_key: | diff --git a/spec/html2rss/web/registry/config_spec.rb b/spec/html2rss/web/registry/config_spec.rb new file mode 100644 index 000000000..e5edd9788 --- /dev/null +++ b/spec/html2rss/web/registry/config_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'climate_control' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Config do + describe '.entry' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'missing-public-key-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + sync: + url: https://registry.test.example/registry-bundle.tar.gz + catalog: true + YAML + end + + after do + FileUtils.rm_f(config_path) + end + + it 'requires a pinned public key for sync-mode registries' do + ClimateControl.modify('REGISTRIES_CONFIG' => config_path) do + expect do + described_class.reload! + described_class.entry('official') + end.to raise_error(Html2rss::Web::Registry::Errors::ConfigError, /requires a pinned public_key/) + end + end + end + + describe 'sync policy parsing' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'sync-policy-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + sync: + channel: html2rss-official + pin_version: v2026.08.22 + max_version: v2026.08.21 + auto_promote: true + allowed_channel_domains: + - phys.org + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- + YAML + end + + after do + FileUtils.rm_f(config_path) + end + + it 'parses sync policy and domain allowlist fields', :aggregate_failures do + ClimateControl.modify('REGISTRIES_CONFIG' => config_path) do + described_class.reload! + entry = described_class.entry('official') + + expect(entry.sync_policy).to have_attributes( + pin_version: 'v2026.08.22', + max_version: 'v2026.08.21', + auto_promote: true + ) + expect(entry.allowed_channel_domains).to eq(['phys.org']) + end + end + + it 'defaults auto_promote to false for security' do + ClimateControl.modify('REGISTRIES_CONFIG' => nil) do + described_class.reload! + entry = described_class.entry('official') + + expect(entry.sync_policy.auto_promote).to be(false) + end + end + end +end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index 3b9cf095b..cf8d91c1f 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'fileutils' require 'spec_helper' require_relative '../../../../app' @@ -40,6 +41,13 @@ expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') end + + it 'prefers the first registry in precedence for duplicate feed ids' do + rows = described_class.current.catalog_rows + apple = rows.find { |row| row.fetch(:id) == 'support.apple.com/en_gb_ht201222' } + + expect(apple.fetch(:registry)).to eq('official') + end end describe '#status' do @@ -53,4 +61,33 @@ ) end end + + describe 'allowed_channel_domains' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + path: spec/fixtures/registries/official + catalog: true + allowed_channel_domains: + - blocked.example + YAML + ENV['REGISTRIES_CONFIG'] = config_path + described_class.reload! + end + + after do + FileUtils.rm_f(config_path) + end + + it 'rejects bundles with channel URLs outside the allowlist' do + expect { described_class.current.config_for('phys.org/weekly') } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /phys.org/) + end + end end diff --git a/spec/html2rss/web/registry/scrape_policy_spec.rb b/spec/html2rss/web/registry/scrape_policy_spec.rb new file mode 100644 index 000000000..681e88cd8 --- /dev/null +++ b/spec/html2rss/web/registry/scrape_policy_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'climate_control' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::ScrapePolicy do + let(:allowed_entry) do + Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :path, + path: nil, + sync_channel: nil, + sync_url: nil, + catalog: true, + public_key_id: 'test', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), + allowed_channel_domains: ['phys.org'] + ) + end + + let(:allowed_bundle) do + Html2rss::Web::Registry::Index::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } + }, + catalog_entries: [] + ) + end + + let(:blocked_bundle) do + Html2rss::Web::Registry::Index::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } }, + 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } + }, + catalog_entries: [] + ) + end + + it 'allows suffix-matching channel domains' do + expect { described_class.enforce!(allowed_entry, allowed_bundle) }.not_to raise_error + end + + it 'raises when a config channel domain is not allowlisted' do + expect { described_class.enforce!(allowed_entry, blocked_bundle) } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) + end +end diff --git a/spec/html2rss/web/registry/store_spec.rb b/spec/html2rss/web/registry/store_spec.rb new file mode 100644 index 000000000..8a1d92508 --- /dev/null +++ b/spec/html2rss/web/registry/store_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require 'digest' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Store do + let(:registry_id) { 'store-test' } + let(:data_root) { File.join(Dir.pwd, 'tmp', 'store-spec-data') } + + before do + ENV['REGISTRY_DATA_ROOT'] = data_root + FileUtils.rm_rf(data_root) + end + + describe '.stage_bundle! and .promote_staged!' do + it 'stages a verified bundle and promotes it to active', :aggregate_failures do + source = build_bundle_dir('stage-source') + active_before = build_bundle_dir('active-before') + + FileUtils.mkdir_p(File.dirname(described_class.registry_dir(registry_id))) + FileUtils.cp_r(active_before, described_class.registry_dir(registry_id)) + + described_class.stage_bundle!(registry_id, source) + + expect(described_class.staged_present?(registry_id)).to be(true) + expect(described_class.staged_version(registry_id)).to eq('stage-source') + expect(described_class.bundle_present?(registry_id)).to be(true) + expect(read_manifest_version(described_class.registry_dir(registry_id))).to eq('active-before') + + described_class.promote_staged!(registry_id) + + expect(described_class.staged_present?(registry_id)).to be(false) + expect(read_manifest_version(described_class.registry_dir(registry_id))).to eq('stage-source') + end + end + + describe '.bundle_present?' do + it 'requires manifest.json instead of any non-empty directory' do + path = described_class.registry_dir(registry_id) + + FileUtils.rm_rf(path) + FileUtils.mkdir_p(path) + File.write(File.join(path, 'placeholder.txt'), 'seed') + + expect(described_class.bundle_present?(registry_id)).to be(false) + + File.write(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE), '{}') + expect(described_class.bundle_present?(registry_id)).to be(true) + end + end + + def build_bundle_dir(version) # rubocop:disable Metrics/MethodLength + dir = Dir.mktmpdir("registry-store-#{version}") + config_path = File.join(dir, 'configs', 'example.com', 'feed.yml') + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, "channel:\n url: https://example.com/\n") + digest = Digest::SHA256.file(config_path).hexdigest + File.write( + File.join(dir, Html2rss::Registry::Manifest::MANIFEST_FILE), + { + format: 'registry.v1', + registry_id: 'store-test', + version:, + public_key_id: 'test', + files: { + 'configs/example.com/feed.yml' => digest + } + }.to_json + ) + dir + end + + def read_manifest_version(path) + JSON.parse(File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE))).fetch('version') + end +end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 4ff1b8877..0630807ec 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'fileutils' +require 'climate_control' require 'spec_helper' require 'webmock/rspec' @@ -43,12 +45,36 @@ end.not_to(change { Html2rss::Web::Registry::Store.bundle_present?('official') }) end - it 'rejects redirects' do + it 'follows bounded redirects to allowed CDN hosts', :aggregate_failures do + cdn_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, download_url) + .to_return(status: 302, headers: { 'Location' => cdn_url }) + stub_request(:get, cdn_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + + status = described_class.run(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + end + + it 'rejects redirects to disallowed hosts' do stub_request(:get, download_url) .to_return(status: 302, headers: { 'Location' => 'https://evil.example/bundle.tar.gz' }) expect { described_class.run(registry_id: 'official') } - .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect/i) + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /host not allowed/i) + end + + it 'rejects excessive redirect chains' do + (1..6).each do |hop| + from = hop == 1 ? download_url : "#{download_url}?hop=#{hop - 1}" + to = "#{download_url}?hop=#{hop}" + stub_request(:get, from).to_return(status: 302, headers: { 'Location' => to }) + end + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect limit/i) end it 'logs signature verification failures to the security logger' do @@ -86,8 +112,82 @@ expect(row).to have_attributes( registry_id: 'official', mode: :sync, - sync_url: 'https://registry.test.example/registry-bundle.tar.gz' + sync_url: 'https://registry.test.example/registry-bundle.tar.gz', + staged_version: nil + ) + end + end + + describe 'sync policy', :registry_sync do + let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } + let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } + let(:policy_config_path) { File.join(Dir.pwd, 'tmp', 'sync-policy-official.yml') } + + before do + FileUtils.mkdir_p(File.dirname(policy_config_path)) + stub_request(:get, download_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + end + + after do + FileUtils.rm_f(policy_config_path) + end + + def write_policy_config(yaml) + File.write(policy_config_path, yaml) + ENV['REGISTRIES_CONFIG'] = policy_config_path + Html2rss::Web::Registry::Index.reload! + end + + it 'stages verified bundles when auto_promote is false', :aggregate_failures do + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml( + download_url:, + auto_promote: false, + sync_extra: { max_version: 'test-fixture' } + ) + ) + + status = described_class.run(registry_id: 'official') + + expect(Html2rss::Web::Registry::Store.staged_present?('official')).to be(true) + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(false) + expect(status.staged_version).to eq('test-fixture') + end + + it 'promotes staged bundles and emits catalog change telemetry', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + allow(Html2rss::Web::Observability).to receive(:emit) + allow(Html2rss::Web::SecurityLogger).to receive(:log_registry_catalog_changed) + + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml(download_url:, auto_promote: false) ) + + described_class.run(registry_id: 'official') + status = described_class.promote_staged!(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.staged_present?('official')).to be(false) + expect(Html2rss::Web::Observability).to have_received(:emit).with( + hash_including(event_name: 'registry.promote_staged', outcome: 'success') + ) + expect(Html2rss::Web::Observability).to have_received(:emit).with( + hash_including(event_name: 'registry.catalog_changed', outcome: 'success') + ) + expect(Html2rss::Web::SecurityLogger).to have_received(:log_registry_catalog_changed) + end + + it 'rejects manifests newer than max_version' do + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml( + download_url:, + auto_promote: true, + sync_extra: { max_version: '0.0.1' } + ) + ) + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /exceeds max_version/) end end diff --git a/spec/html2rss/web/registry/sync_url_resolver_spec.rb b/spec/html2rss/web/registry/sync_url_resolver_spec.rb new file mode 100644 index 000000000..1736a8e45 --- /dev/null +++ b/spec/html2rss/web/registry/sync_url_resolver_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'webmock/rspec' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::SyncUrlResolver do + describe '.resolve' do + it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + tag_api = format( + described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, + tag: 'v2026.08.22' + ) + download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, tag_api).to_return( + status: 200, + body: { + assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] + }.to_json + ) + + entry = Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :sync, + path: nil, + sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, + sync_url: nil, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), + allowed_channel_domains: [] + ) + + expect(described_class.resolve(entry)).to eq(download_url) + end + end +end diff --git a/spec/html2rss/web/registry/version_gate_spec.rb b/spec/html2rss/web/registry/version_gate_spec.rb new file mode 100644 index 000000000..09b7d43dd --- /dev/null +++ b/spec/html2rss/web/registry/version_gate_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::VersionGate do + it 'compares dotted numeric versions' do + expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) + expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) + end + + it 'strips a leading v prefix before comparing' do + expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) + end + + it 'returns false when max_version is unset' do + expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + end +end diff --git a/spec/smoke/docker_spec.rb b/spec/smoke/docker_spec.rb index 115d2b76b..c056a2624 100644 --- a/spec/smoke/docker_spec.rb +++ b/spec/smoke/docker_spec.rb @@ -112,4 +112,13 @@ def expect_json_feed_response(path) expect(body.dig('error', 'code')).to eq('FORBIDDEN') expect(body.dig('error', 'message')).to eq('Auto source feature is disabled') end + + it 'exposes the config catalog without authentication', :aggregate_failures do + response, payload = get_json('/api/v1/configs') + + expect(response).to be_a(Net::HTTPOK) + expect(payload.fetch('success')).to be(true) + expect(payload.dig('data', 'configs')).to be_an(Array) + expect(payload.dig('meta', 'catalog_version')).to eq(1) + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4a7811d74..383238f13 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,13 +8,17 @@ require 'simplecov' SimpleCov.start do - add_filter '/spec/' - add_filter '/config/' + enable_coverage :branch + primary_coverage :branch - track_files '**/*.rb' + add_group 'App', 'app' + add_group 'Config', 'config' - minimum_coverage 80 unless ENV['OPENAPI'] - maximum_coverage_drop 5 unless ENV['OPENAPI'] + skip_filter %r{/spec/} + skip_filter %r{/config/} + + minimum_coverage line: 80, branch: 70 unless ENV['OPENAPI'] + maximum_coverage_drop line: 5, branch: 5 unless ENV['OPENAPI'] end end diff --git a/spec/support/openapi.rb b/spec/support/openapi.rb index 30ea6e7e1..13fa7edf0 100644 --- a/spec/support/openapi.rb +++ b/spec/support/openapi.rb @@ -233,5 +233,11 @@ else spec[:tags] = tags end + + paths = spec['paths'] || spec[:paths] + catalog_configs_required = paths&.dig('/configs', 'get', 'responses', '200', 'content', + 'application/json', 'schema', 'properties', 'data', 'properties', 'configs', + 'items', 'required') + catalog_configs_required&.delete('registry') if catalog_configs_required.is_a?(Array) end end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index b0f6a1dca..839d9e13a 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -1,15 +1,14 @@ # frozen_string_literal: true -require 'openssl' require 'fileutils' -require 'zlib' +require 'openssl' require 'stringio' -require 'rubygems/package' +require 'yaml' module RegistrySyncTestHelpers - HTML2RSS_FIXTURES = File.expand_path('../../../html2rss/spec/fixtures/registry', __dir__) - TEST_PUBLIC_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pub')) - TEST_PRIVATE_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pem')) + FIXTURE_KEYS_ROOT = File.expand_path('../fixtures/registries/keys', __dir__) + TEST_PUBLIC_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pub')) + TEST_PRIVATE_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pem')) SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') @@ -19,39 +18,57 @@ module RegistrySyncTestHelpers def configure_sync_registry! ENV['REGISTRIES_CONFIG'] = SYNC_REGISTRIES_CONFIG ENV['REGISTRY_DATA_ROOT'] = SYNC_DATA_ROOT - ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example' + ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example,release-assets.githubusercontent.com' FileUtils.rm_rf(SYNC_DATA_ROOT) Html2rss::Web::Registry::Index.reload! end - def build_signed_tarball # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def build_signed_tarball bundle_dir = Dir.mktmpdir('signed-registry-bundle') FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) - private_key = OpenSSL::PKey.read(TEST_PRIVATE_KEY) - signature = private_key.sign(nil, manifest.canonical_bytes) - File.write( - File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), - [signature].pack('m0') - ) + sign_manifest!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) - tar_io = StringIO.new - Gem::Package::TarWriter.new(tar_io) do |tar| - Dir.glob(File.join(bundle_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| - next if File.directory?(path) + pack_bundle_dir(bundle_dir) + ensure + FileUtils.rm_rf(bundle_dir) + end - relative = path.delete_prefix("#{bundle_dir}/") - tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } - end - end + def pack_bundle_dir(bundle_dir) + dir = File.dirname(tarball_path = File.join(Dir.mktmpdir('registry-sync-tarball'), 'bundle.tar.gz')) + env = { 'COPYFILE_DISABLE' => '1' } + success = system(env, 'tar', '--format=ustar', '-czf', tarball_path, '-C', bundle_dir, '.', exception: false) + raise "Failed to pack registry test bundle from #{bundle_dir}" unless success - gz_io = StringIO.new - Zlib::GzipWriter.wrap(gz_io) { |gz| gz.write(tar_io.string) } - gz_io.string + File.binread(tarball_path) ensure - FileUtils.rm_rf(bundle_dir) + FileUtils.rm_rf(dir) if dir + end + + def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop:disable Metrics/MethodLength + sync = { 'url' => download_url }.merge(sync_extra.transform_keys(&:to_s)) + YAML.dump( + { + 'precedence' => ['official'], + 'registries' => { + 'official' => { + 'sync' => sync, + 'auto_promote' => auto_promote, + 'catalog' => true, + 'public_key_id' => 'test-key', + 'public_key' => TEST_PUBLIC_KEY + } + } + } + ) + end + + def sign_manifest!(manifest, key_pem:, bundle_dir:) + private_key = OpenSSL::PKey.read(key_pem) + signature = private_key.sign(nil, manifest.canonical_bytes) + File.write(File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), [signature].pack('m0')) end end From 077917e818acddc88d3936ab36e379919f309e02 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:23:23 +0200 Subject: [PATCH 07/19] chore(deps): pin html2rss to registry hardening commit Align Gemfile.lock with feat/registry-v1 b32ec095 for signed bundle verify and manifest-only catalog APIs. --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 37331d84e..d0b5bcedb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: e32014ac089679cac7e91fd55f3bd85a0489771e + revision: b32ec095bd366075297f9b7f59cf55db6b665d2a branch: feat/registry-v1 specs: html2rss (0.27.1) From f03730cacc41530b3a5a28ba6d42bc3d80b073f3 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:16:06 +0200 Subject: [PATCH 08/19] fix(ci): use SimpleCov add_filter and git html2rss pin Replace invalid skip_filter with add_filter so ruby and openapi CI jobs load spec_helper; keep html2rss on feat/registry-v1 until gem release. --- Gemfile | 2 +- spec/spec_helper.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 8ff642c87..85e42a7e8 100644 --- a/Gemfile +++ b/Gemfile @@ -4,8 +4,8 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } +# Until rubygems 0.28.0: git branch; local monorepo: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' -# Local monorepo override: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'base64' gem 'rack-cache' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 383238f13..25b4b1d90 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,8 +14,8 @@ add_group 'App', 'app' add_group 'Config', 'config' - skip_filter %r{/spec/} - skip_filter %r{/config/} + add_filter %r{/spec/} + add_filter %r{/config/} minimum_coverage line: 80, branch: 70 unless ENV['OPENAPI'] maximum_coverage_drop line: 5, branch: 5 unless ENV['OPENAPI'] From 4aecf78b5b4e6d4e4bdf43e0986c3add8cb4140e Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:49:18 +0200 Subject: [PATCH 09/19] refactor(registry): fold sync satellites into sync_transport --- app/web/registry/catalog_change_reporter.rb | 96 ----------------- app/web/registry/config.rb | 2 +- app/web/registry/sync.rb | 101 +++++++++++++++--- app/web/registry/sync_coordinator.rb | 23 ---- .../{sync_fetcher.rb => sync_transport.rb} | 101 +++++++++++++++++- app/web/registry/sync_url_resolver.rb | 80 -------------- app/web/registry/version_gate.rb | 40 ------- spec/html2rss/web/registry/sync_spec.rb | 48 +++++++++ .../web/registry/sync_url_resolver_spec.rb | 39 ------- .../web/registry/version_gate_spec.rb | 20 ---- 10 files changed, 235 insertions(+), 315 deletions(-) delete mode 100644 app/web/registry/catalog_change_reporter.rb delete mode 100644 app/web/registry/sync_coordinator.rb rename app/web/registry/{sync_fetcher.rb => sync_transport.rb} (52%) delete mode 100644 app/web/registry/sync_url_resolver.rb delete mode 100644 app/web/registry/version_gate.rb delete mode 100644 spec/html2rss/web/registry/sync_url_resolver_spec.rb delete mode 100644 spec/html2rss/web/registry/version_gate_spec.rb diff --git a/app/web/registry/catalog_change_reporter.rb b/app/web/registry/catalog_change_reporter.rb deleted file mode 100644 index f218090c3..000000000 --- a/app/web/registry/catalog_change_reporter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Reports catalog diffs after a registry bundle is promoted to active. - module CatalogChangeReporter - module_function - - ## - # @param registry_id [String] - # @param previous_bundle [Index::RegistryBundle, nil] - # @param new_bundle [Index::RegistryBundle] - # @return [void] - def report!(registry_id:, previous_bundle:, new_bundle:) - diff = build_diff(previous_bundle, new_bundle) - return if diff.empty? - - emit_observability!(registry_id, diff) - emit_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? - end - - ## - # @param previous_bundle [Index::RegistryBundle, nil] - # @param new_bundle [Index::RegistryBundle] - # @return [Hash{Symbol => Object}] - def build_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength - previous_version = previous_bundle&.manifest&.version - new_version = new_bundle.manifest.version - previous_ids = catalog_ids(previous_bundle) - new_ids = catalog_ids(new_bundle) - added = new_ids - previous_ids - removed = previous_ids - new_ids - - return {} if previous_version == new_version && added.empty? && removed.empty? - - { - previous_version:, - version: new_version, - added:, - removed: - } - end - - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength - Observability.emit( - event_name: 'registry.catalog_changed', - outcome: 'success', - level: :warn, - details: { - registry_id:, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - } - ) - end - - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_security!(registry_id, diff) - SecurityLogger.log_registry_catalog_changed( - registry_id, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - ) - end - - ## - # @param bundle [Index::RegistryBundle, nil] - # @return [Array] - def catalog_ids(bundle) - return [] unless bundle - - bundle.catalog_entries.map(&:id) - end - - private_class_method :build_diff, :emit_observability!, :emit_security!, :catalog_ids - end - end - end -end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 065fcfae5..789bd07d6 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -194,7 +194,7 @@ def build_sync_entry(registry_id, raw, sync) def resolved_sync_url(sync_url, sync_channel) return sync_url if sync_url && !sync_url.empty? - SyncUrlResolver.resolve_channel_url(sync_channel) + SyncTransport.resolve_channel_url(sync_channel) end ## diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index f96c3448e..9c110e915 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -21,6 +21,7 @@ module Sync # rubocop:disable Metrics/ModuleLength @boot_mutex = Mutex.new @boot_started = false @timer_started = false + REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } class << self # rubocop:disable Metrics/ClassLength ## @@ -32,7 +33,7 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - SyncUrlResolver.resolve(entry) + SyncTransport.resolve(entry) end ## @@ -42,7 +43,7 @@ def sync_url_for(registry_id) # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] def run(registry_id:, dry_run: false) - SyncCoordinator.run(registry_id) { run!(registry_id:, dry_run:) } + with_registry_lock(registry_id) { run!(registry_id:, dry_run:) } end ## @@ -51,7 +52,7 @@ def run(registry_id:, dry_run: false) # @param registry_id [String, Symbol] # @return [SyncStatus] def promote_staged!(registry_id:) - SyncCoordinator.run(registry_id) { promote_staged_bundle!(registry_id:) } + with_registry_lock(registry_id) { promote_staged_bundle!(registry_id:) } end ## @@ -129,6 +130,14 @@ def unusable_sync_registries private + ## + # @param registry_id [String, Symbol] + # @yield runs sync work exclusively for the registry id + # @return [Object] block result + def with_registry_lock(registry_id, &) + REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) + end + ## # @param registry_id [String, Symbol] # @param dry_run [Boolean] @@ -140,7 +149,7 @@ def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/Meth end staging_root = nil - download_url = SyncUrlResolver.resolve(entry) + download_url = SyncTransport.resolve(entry) staging_root, staged_dir = fetch_and_verify!(entry, download_url) unless dry_run if entry.sync_policy.auto_promote @@ -199,20 +208,88 @@ def activate_bundle!(registry_id, staged_dir) end ## - # @param registry_id [String, Symbol] + # @param registry_id [String] # @param previous_bundle [Index::RegistryBundle, nil] # @return [void] def report_catalog_change!(registry_id, previous_bundle) new_bundle = Index.current.bundle_for(registry_id) return unless new_bundle - CatalogChangeReporter.report!( - registry_id: registry_id.to_s, - previous_bundle:, - new_bundle: + diff = build_catalog_diff(previous_bundle, new_bundle) + return if diff.empty? + + emit_catalog_change_observability!(registry_id, diff) + emit_catalog_change_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? + end + + ## + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [Hash{Symbol => Object}] + def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + previous_version = previous_bundle&.manifest&.version + new_version = new_bundle.manifest.version + previous_ids = catalog_ids(previous_bundle) + new_ids = catalog_ids(new_bundle) + added = new_ids - previous_ids + removed = previous_ids - new_ids + + return {} if previous_version == new_version && added.empty? && removed.empty? + + { + previous_version:, + version: new_version, + added:, + removed: + } + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_catalog_change_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength + Observability.emit( + event_name: 'registry.catalog_changed', + outcome: 'success', + level: :warn, + details: { + registry_id:, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + } ) end + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_catalog_change_security!(registry_id, diff) + SecurityLogger.log_registry_catalog_changed( + registry_id, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + ) + end + + ## + # @param bundle [Index::RegistryBundle, nil] + # @return [Array] + def catalog_ids(bundle) + return [] unless bundle + + bundle.catalog_entries.map(&:id) + end + ## # @return [Boolean] def skip_boot? @@ -285,7 +362,7 @@ def background_jitter_seconds(interval_hours) # @param download_url [String] # @return [Array(String, String)] staging root and verified bundle directory def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tarball = SyncFetcher.fetch!(download_url) + tarball = SyncTransport.fetch!(download_url) staging_root = Dir.mktmpdir('registry-sync-') staged_dir = File.join(staging_root, 'bundle') tarball_path = File.join(staging_root, 'download.tar.gz') @@ -334,7 +411,7 @@ def enforce_max_version!(entry, staged_dir) max_version = entry.sync_policy.max_version return unless max_version - return unless VersionGate.exceeds_max?(manifest.version, max_version) + return unless SyncTransport.exceeds_max?(manifest.version, max_version) raise Errors::SyncError, "Registry '#{entry.id}' manifest version '#{manifest.version}' exceeds max_version '#{max_version}'" @@ -394,7 +471,7 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength version: row.version, staged_version: Store.staged_version(row.id), updated_at: row.updated_at, - sync_url: entry.mode == :sync ? SyncUrlResolver.resolve(entry) : nil, + sync_url: entry.mode == :sync ? SyncTransport.resolve(entry) : nil, last_error: state.last_error ) end diff --git a/app/web/registry/sync_coordinator.rb b/app/web/registry/sync_coordinator.rb deleted file mode 100644 index 2bd641e07..000000000 --- a/app/web/registry/sync_coordinator.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Serializes sync work per registry id to avoid concurrent bundle swaps. - module SyncCoordinator - REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - - module_function - - ## - # @param registry_id [String, Symbol] - # @yield runs sync work exclusively for the registry id - # @return [Object] block result - def run(registry_id, &) - REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) - end - end - end - end -end diff --git a/app/web/registry/sync_fetcher.rb b/app/web/registry/sync_transport.rb similarity index 52% rename from app/web/registry/sync_fetcher.rb rename to app/web/registry/sync_transport.rb index 6d13712a3..d67461a59 100644 --- a/app/web/registry/sync_fetcher.rb +++ b/app/web/registry/sync_transport.rb @@ -2,13 +2,14 @@ require 'net/http' require 'uri' +require 'json' module Html2rss module Web module Registry ## - # HTTPS fetcher with host allowlisting and bounded redirect following. - module SyncFetcher + # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. + module SyncTransport # rubocop:disable Metrics/ModuleLength DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -20,6 +21,12 @@ module SyncFetcher MAX_RESPONSE_BYTES = 52_428_800 DEFAULT_MAX_REDIRECTS = 5 + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_GITHUB_TAG_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do ## # @return [RedirectPolicy] @@ -30,7 +37,7 @@ def self.default ## # @return [Array] def self.default_allowed_hosts - DEFAULT_ALLOWED_HOSTS + SyncFetcher.extra_allowed_hosts + DEFAULT_ALLOWED_HOSTS + SyncTransport.extra_allowed_hosts end end @@ -45,6 +52,42 @@ def fetch!(url, policy: RedirectPolicy.default) follow_redirects!(uri, policy) end + ## + # @param entry [Entry] + # @return [String] + def resolve(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + + pin_version = entry.sync_policy.pin_version + if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + + return resolve_official_download_url + end + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param manifest_version [String] + # @param max_version [String, nil] + # @return [Boolean] true when manifest_version is newer than max_version + def exceeds_max?(manifest_version, max_version) + return false if max_version.nil? || max_version.empty? + + compare(normalize(manifest_version), normalize(max_version)).positive? + end + ## # @param uri [URI::HTTPS] # @param policy [RedirectPolicy] @@ -82,6 +125,56 @@ def extra_allowed_hosts .reject(&:empty?) end + ## + # @param tag [String] + # @return [String] + def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + response_body = fetch!(api_url) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + unless url + raise Errors::SyncError, + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + end + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @param version [String] + # @return [String] + def normalize(version) + version.to_s.delete_prefix('v') + end + + ## + # @param left [String] + # @param right [String] + # @return [Integer] + def compare(left, right) + Gem::Version.new(left) <=> Gem::Version.new(right) + rescue ArgumentError + left <=> right + end + ## # @param url [String] # @return [URI::HTTPS] @@ -144,7 +237,7 @@ def read_body!(response) body end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/registry/sync_url_resolver.rb b/app/web/registry/sync_url_resolver.rb deleted file mode 100644 index d035c7396..000000000 --- a/app/web/registry/sync_url_resolver.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module Html2rss - module Web - module Registry - ## - # Resolves registry sync download URLs from config entries and official release metadata. - module SyncUrlResolver - OFFICIAL_GITHUB_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' - OFFICIAL_GITHUB_TAG_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' - OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' - - module_function - - ## - # @param entry [Entry] - # @return [String] - def resolve(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? - - pin_version = entry.sync_policy.pin_version - if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? - - return resolve_official_download_url - end - - raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" - end - - ## - # @param sync_channel [String, nil] - # @return [String] - def resolve_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - - raise Errors::ConfigError, "Unknown sync channel '#{channel}'" - end - - ## - # @param tag [String] - # @return [String] - def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength - api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) - response_body = SyncFetcher.fetch!(api_url) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - unless url - raise Errors::SyncError, - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" - end - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - - ## - # @return [String] - def resolve_official_download_url - response_body = SyncFetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - end - end - end -end diff --git a/app/web/registry/version_gate.rb b/app/web/registry/version_gate.rb deleted file mode 100644 index 27f6aadbd..000000000 --- a/app/web/registry/version_gate.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Compares registry manifest versions for sync policy caps. - module VersionGate - module_function - - ## - # @param manifest_version [String] - # @param max_version [String, nil] - # @return [Boolean] true when manifest_version is newer than max_version - def exceeds_max?(manifest_version, max_version) - return false if max_version.nil? || max_version.empty? - - compare(normalize(manifest_version), normalize(max_version)).positive? - end - - ## - # @param version [String] - # @return [String] - def normalize(version) - version.to_s.delete_prefix('v') - end - - ## - # @param left [String] - # @param right [String] - # @return [Integer] - def compare(left, right) - Gem::Version.new(left) <=> Gem::Version.new(right) - rescue ArgumentError - left <=> right - end - end - end - end -end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 0630807ec..20245c131 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -216,4 +216,52 @@ def write_policy_config(yaml) expect(described_class).not_to have_received(:seed_registry!) end end + + describe Html2rss::Web::Registry::SyncTransport do + describe '.exceeds_max?' do + it 'compares dotted numeric versions' do + expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) + expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) + end + + it 'strips a leading v prefix before comparing' do + expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) + end + + it 'returns false when max_version is unset' do + expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + end + end + + describe '.resolve' do + it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + tag_api = format( + described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, + tag: 'v2026.08.22' + ) + download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, tag_api).to_return( + status: 200, + body: { + assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] + }.to_json + ) + + entry = Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :sync, + path: nil, + sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, + sync_url: nil, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), + allowed_channel_domains: [] + ) + + expect(described_class.resolve(entry)).to eq(download_url) + end + end + end end diff --git a/spec/html2rss/web/registry/sync_url_resolver_spec.rb b/spec/html2rss/web/registry/sync_url_resolver_spec.rb deleted file mode 100644 index 1736a8e45..000000000 --- a/spec/html2rss/web/registry/sync_url_resolver_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'webmock/rspec' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::SyncUrlResolver do - describe '.resolve' do - it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength - tag_api = format( - described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, - tag: 'v2026.08.22' - ) - download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' - stub_request(:get, tag_api).to_return( - status: 200, - body: { - assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] - }.to_json - ) - - entry = Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :sync, - path: nil, - sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, - sync_url: nil, - catalog: true, - public_key_id: 'html2rss:registry:2026', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), - allowed_channel_domains: [] - ) - - expect(described_class.resolve(entry)).to eq(download_url) - end - end -end diff --git a/spec/html2rss/web/registry/version_gate_spec.rb b/spec/html2rss/web/registry/version_gate_spec.rb deleted file mode 100644 index 09b7d43dd..000000000 --- a/spec/html2rss/web/registry/version_gate_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::VersionGate do - it 'compares dotted numeric versions' do - expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) - expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) - end - - it 'strips a leading v prefix before comparing' do - expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) - end - - it 'returns false when max_version is unset' do - expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) - end -end From 4a091e6170da390a6d541c663bbb0c2cb70b64d1 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:50:35 +0200 Subject: [PATCH 10/19] refactor(registry): fold catalog wire and load guards into index --- app/web/registry/index.rb | 141 +++++++++++++++++- app/web/registry/local_catalog.rb | 75 ---------- app/web/registry/local_catalog_row.rb | 18 --- app/web/registry/registry_catalog_row.rb | 35 ----- app/web/registry/scrape_policy.rb | 58 ------- app/web/registry/trust_context.rb | 54 ------- spec/html2rss/web/registry/index_spec.rb | 78 +++++++++- .../web/registry/scrape_policy_spec.rb | 56 ------- 8 files changed, 207 insertions(+), 308 deletions(-) delete mode 100644 app/web/registry/local_catalog.rb delete mode 100644 app/web/registry/local_catalog_row.rb delete mode 100644 app/web/registry/registry_catalog_row.rb delete mode 100644 app/web/registry/scrape_policy.rb delete mode 100644 app/web/registry/trust_context.rb delete mode 100644 spec/html2rss/web/registry/scrape_policy_spec.rb diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 5c0df5d84..1fbbabbbd 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'uri' + module Html2rss module Web module Registry @@ -9,6 +11,60 @@ class Index # rubocop:disable Metrics/ClassLength RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) + CatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source, + :registry + ) do + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [CatalogRow] + def self.from_entry(entry, registry_id) + new( + id: entry.id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + + ## + # @param feed_name [String, Symbol] + # @param feed_config [Hash{Symbol => Object}] + # @return [CatalogRow, nil] + def self.from_local_feed(feed_name, feed_config) # rubocop:disable Metrics/MethodLength + directory = feed_config[:directory] || {} + title = directory[:title] + return nil if title.to_s.strip.empty? + + id = feed_name.to_s + channel = feed_config[:channel] || {} + new( + id:, + path: "/#{id}.rss", + source: 'local', + directory: Html2rss::Registry::CatalogBuilder.directory_payload(directory, title), + channel: Html2rss::Registry::CatalogBuilder.channel_payload(channel, title), + parameters: { schema: {}, defaults: {} }, + registry: nil + ) + end + + ## + # @return [Hash{Symbol => Object}] + def to_h + super.compact + end + end + @mutex = Mutex.new @current = nil @@ -50,7 +106,7 @@ def config_for(feed_id) # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows rows = registry_catalog_rows - LocalCatalog.rows.each { |row| rows[row.id] = row } + local_catalog_rows.each { |row| rows[row.id] = row } rows.values.sort_by(&:id).map(&:to_h) end @@ -79,7 +135,7 @@ def status private ## - # @return [Hash{String => RegistryCatalogRow, LocalCatalogRow}] + # @return [Hash{String => CatalogRow}] def registry_catalog_rows Config.precedence.each_with_object({}) do |registry_id, rows| next unless Config.catalog_enabled?(registry_id) @@ -88,11 +144,19 @@ def registry_catalog_rows next unless bundle bundle.catalog_entries.each do |entry| - rows[entry.id] ||= RegistryCatalogRow.from_entry(entry, registry_id) + rows[entry.id] ||= CatalogRow.from_entry(entry, registry_id) end end end + ## + # @return [Array] + def local_catalog_rows + LocalConfig.feeds.filter_map do |feed_name, feed_config| + CatalogRow.from_local_feed(feed_name, feed_config) + end + end + ## # @return [Hash{String => RegistryBundle}] def loaded_bundles @@ -110,15 +174,82 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength bundle = Html2rss::Registry::Bundle.load( directory, - **TrustContext.for_entry(entry, directory).load_options + **trust_options_for(entry, directory) ) registry_bundle = to_registry_bundle(registry_id, bundle) - ScrapePolicy.enforce!(entry, registry_bundle) + enforce_scrape_policy!(entry, registry_bundle) registry_bundle rescue Html2rss::Registry::Error => error raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end + ## + # @param entry [Entry] + # @param bundle_dir [String] + # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} + def trust_options_for(entry, bundle_dir) + case entry.mode + in :path + { trust: :integrity_only, public_keys: {} } + in :sync + if signed_bundle?(bundle_dir, entry) + { trust: :signed, public_keys: entry.public_keys } + else + { trust: :integrity_only, public_keys: {} } + end + end + end + + ## + # @param bundle_dir [String] + # @param entry [Entry] + # @return [Boolean] + def signed_bundle?(bundle_dir, entry) + signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) + File.file?(signature_path) && !entry.public_keys.empty? + end + + ## + # @param entry [Entry] + # @param bundle [RegistryBundle] + # @return [void] + # @raise [Errors::LoadError] when a config channel URL violates the allowlist + def enforce_scrape_policy!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + allowed = entry.allowed_channel_domains + return if allowed.nil? || allowed.empty? + + bundle.configs.each do |feed_id, config| + channel_url = config.dig(:channel, :url) + host = channel_host_for(channel_url) + next if host && allowed.any? { |domain| channel_domain_allowed?(host, domain) } + + raise Errors::LoadError, + "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ + "'#{host || channel_url}' is not allowed by allowed_channel_domains" + end + end + + ## + # @param host [String] + # @param allowed_domain [String] + # @return [Boolean] + def channel_domain_allowed?(host, allowed_domain) + normalized_host = host.downcase + normalized_domain = allowed_domain.downcase + normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") + end + + ## + # @param channel_url [String, nil] + # @return [String, nil] + def channel_host_for(channel_url) + return nil if channel_url.to_s.strip.empty? + + URI.parse(channel_url).host + rescue URI::InvalidURIError + nil + end + ## # @param registry_id [String] # @param bundle [Html2rss::Registry::Bundle::BundleData] diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb deleted file mode 100644 index 9623bf45f..000000000 --- a/app/web/registry/local_catalog.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Builds catalog wire rows for local feeds.yml entries. - module LocalCatalog - module_function - - ## - # @return [Array] - def rows - LocalConfig.feeds.filter_map do |feed_name, feed_config| - build_row(feed_name, feed_config) - end - end - - ## - # @param feed_name [String, Symbol] - # @param feed_config [Hash{Symbol => Object}] - # @return [LocalCatalogRow, nil] - def build_row(feed_name, feed_config) - directory = feed_config[:directory] || {} - title = directory[:title] - return nil if title.to_s.strip.empty? - - row_identity(feed_name, directory, title, feed_config[:channel] || {}) - end - - ## - # @param feed_name [String, Symbol] - # @param directory [Hash{Symbol => Object}] - # @param title [String] - # @param channel [Hash{Symbol => Object}] - # @return [LocalCatalogRow] - def row_identity(feed_name, directory, title, channel) - id = feed_name.to_s - LocalCatalogRow.new( - id:, - path: "/#{id}.rss", - source: 'local', - directory: directory_payload(directory, title), - channel: channel_payload(channel, title), - parameters: { schema: {}, defaults: {} } - ) - end - - ## - # @param directory [Hash{Symbol => Object}] - # @param title [String] - # @return [Hash{Symbol => Object}] - def directory_payload(directory, title) - { - title: title.to_s, - summary: directory[:summary], - topics: Array(directory[:topics]) - }.compact - end - - ## - # @param channel [Hash{Symbol => Object}] - # @param title [String] - # @return [Hash{Symbol => Object}] - def channel_payload(channel, title) - { - url: channel.fetch(:url), - language: channel[:language], - title: channel[:title] || title.to_s - }.compact - end - end - end - end -end diff --git a/app/web/registry/local_catalog_row.rb b/app/web/registry/local_catalog_row.rb deleted file mode 100644 index 0d95a784f..000000000 --- a/app/web/registry/local_catalog_row.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Catalog wire row for local feeds.yml entries. - LocalCatalogRow = Data.define( - :id, - :path, - :directory, - :channel, - :parameters, - :source - ) - end - end -end diff --git a/app/web/registry/registry_catalog_row.rb b/app/web/registry/registry_catalog_row.rb deleted file mode 100644 index 27d3791e6..000000000 --- a/app/web/registry/registry_catalog_row.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Catalog wire row for registry-sourced feed configs. - RegistryCatalogRow = Data.define( - :id, - :path, - :directory, - :channel, - :parameters, - :source, - :registry - ) do - ## - # @param entry [Html2rss::Registry::CatalogEntry] - # @param registry_id [String] - # @return [RegistryCatalogRow] - def self.from_entry(entry, registry_id) - new( - id: entry.id, - path: entry.path, - directory: entry.directory, - channel: entry.channel, - parameters: entry.parameters, - source: 'registry', - registry: registry_id - ) - end - end - end - end -end diff --git a/app/web/registry/scrape_policy.rb b/app/web/registry/scrape_policy.rb deleted file mode 100644 index b341d751f..000000000 --- a/app/web/registry/scrape_policy.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -module Html2rss - module Web - module Registry - ## - # Enforces optional channel URL domain allowlists for registry bundles. - module ScrapePolicy - module_function - - ## - # @param entry [Entry] - # @param bundle [Index::RegistryBundle] - # @return [void] - # @raise [Errors::LoadError] when a config channel URL violates the allowlist - def enforce!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity - allowed = entry.allowed_channel_domains - return if allowed.nil? || allowed.empty? - - bundle.configs.each do |feed_id, config| - channel_url = config.dig(:channel, :url) - host = host_for(channel_url) - next if host && allowed.any? { |domain| domain_allowed?(host, domain) } - - raise Errors::LoadError, - "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ - "'#{host || channel_url}' is not allowed by allowed_channel_domains" - end - end - - ## - # @param host [String] - # @param allowed_domain [String] - # @return [Boolean] - def domain_allowed?(host, allowed_domain) - normalized_host = host.downcase - normalized_domain = allowed_domain.downcase - normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") - end - - ## - # @param channel_url [String, nil] - # @return [String, nil] - def host_for(channel_url) - return nil if channel_url.to_s.strip.empty? - - URI.parse(channel_url).host - rescue URI::InvalidURIError - nil - end - - private_class_method :host_for - end - end - end -end diff --git a/app/web/registry/trust_context.rb b/app/web/registry/trust_context.rb deleted file mode 100644 index 960593a10..000000000 --- a/app/web/registry/trust_context.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Trust mode and verification keys passed to {Html2rss::Registry::Bundle.load}. - TrustContext = Data.define(:mode, :public_keys) do - ## - # @param entry [Entry] - # @param bundle_dir [String] - # @return [TrustContext] - def self.for_entry(entry, bundle_dir) - case entry.mode - in :path - integrity_only - in :sync - signed_bundle?(bundle_dir, entry) ? signed(entry.public_keys) : integrity_only - end - end - - ## - # @return [TrustContext] - def self.integrity_only - new(mode: :integrity_only, public_keys: {}) - end - - ## - # @param public_keys [Hash{String => OpenSSL::PKey::PKey}] - # @return [TrustContext] - def self.signed(public_keys) - new(mode: :signed, public_keys:) - end - - ## - # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} - def load_options - { trust: mode, public_keys: } - end - - ## - # @param bundle_dir [String] - # @param entry [Entry] - # @return [Boolean] - def self.signed_bundle?(bundle_dir, entry) - signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) - File.file?(signature_path) && !entry.public_keys.empty? - end - - private_class_method :signed_bundle? - end - end - end -end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index cf8d91c1f..f5797ece8 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -48,6 +48,26 @@ expect(apple.fetch(:registry)).to eq('official') end + + it 'merges local feeds.yml rows after registry rows', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + allow(Html2rss::Web::LocalConfig).to receive(:feeds).and_return( + 'team/releases' => { + directory: { title: 'Team Releases', summary: 'Internal release notes' }, + channel: { url: 'https://team.example/releases', title: 'Team Releases' } + } + ) + + rows = described_class.current.catalog_rows + local = rows.find { |row| row.fetch(:id) == 'team/releases' } + + expect(local).to include( + source: 'local', + path: '/team/releases.rss', + directory: hash_including(title: 'Team Releases'), + channel: hash_including(url: 'https://team.example/releases') + ) + expect(local).not_to have_key(:registry) + end end describe '#status' do @@ -62,10 +82,60 @@ end end + describe 'channel domain allowlist' do + let(:index) { described_class.new } + let(:entry) do + Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :path, + path: nil, + sync_channel: nil, + sync_url: nil, + catalog: true, + public_key_id: 'test', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), + allowed_channel_domains: ['phys.org'] + ) + end + let(:allowed_bundle) do + described_class::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } + }, + catalog_entries: [] + ) + end + + it 'allows suffix-matching channel domains' do + expect { index.send(:enforce_scrape_policy!, entry, allowed_bundle) }.not_to raise_error + end + + it 'raises when a config channel domain is not allowlisted' do + blocked_bundle = described_class::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } + }, + catalog_entries: [] + ) + + expect { index.send(:enforce_scrape_policy!, entry, blocked_bundle) } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) + end + end + describe 'allowed_channel_domains' do let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } - before do + after do + FileUtils.rm_f(config_path) + end + + it 'rejects bundles with channel URLs outside the allowlist' do # rubocop:disable RSpec/ExampleLength FileUtils.mkdir_p(File.dirname(config_path)) File.write(config_path, <<~YAML) precedence: @@ -79,13 +149,7 @@ YAML ENV['REGISTRIES_CONFIG'] = config_path described_class.reload! - end - - after do - FileUtils.rm_f(config_path) - end - it 'rejects bundles with channel URLs outside the allowlist' do expect { described_class.current.config_for('phys.org/weekly') } .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /phys.org/) end diff --git a/spec/html2rss/web/registry/scrape_policy_spec.rb b/spec/html2rss/web/registry/scrape_policy_spec.rb deleted file mode 100644 index 681e88cd8..000000000 --- a/spec/html2rss/web/registry/scrape_policy_spec.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' -require 'climate_control' -require 'spec_helper' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::ScrapePolicy do - let(:allowed_entry) do - Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :path, - path: nil, - sync_channel: nil, - sync_url: nil, - catalog: true, - public_key_id: 'test', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), - allowed_channel_domains: ['phys.org'] - ) - end - - let(:allowed_bundle) do - Html2rss::Web::Registry::Index::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } - }, - catalog_entries: [] - ) - end - - let(:blocked_bundle) do - Html2rss::Web::Registry::Index::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } }, - 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } - }, - catalog_entries: [] - ) - end - - it 'allows suffix-matching channel domains' do - expect { described_class.enforce!(allowed_entry, allowed_bundle) }.not_to raise_error - end - - it 'raises when a config channel domain is not allowlisted' do - expect { described_class.enforce!(allowed_entry, blocked_bundle) } - .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) - end -end From 9b79bd7ee10a422bdf40a3dd6197841ec452bcbc Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:50:55 +0200 Subject: [PATCH 11/19] refactor(registry): centralize manifest I/O and dedup test signing --- app/web/registry/index.rb | 3 ++- app/web/registry/store.rb | 28 +++++++++++++++++----------- app/web/registry/sync.rb | 5 ++--- spec/support/registry_sync.rb | 9 +-------- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 1fbbabbbd..edea52fa4 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -172,6 +172,7 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength return nil unless directory && File.directory?(directory) return nil unless active_bundle_present?(directory) + # Verify on read: disk-trust seam; Sync already verifies downloaded bundles before promote. bundle = Html2rss::Registry::Bundle.load( directory, **trust_options_for(entry, directory) @@ -276,7 +277,7 @@ def bundle_directory(entry) # @param directory [String] # @return [Boolean] def active_bundle_present?(directory) - File.file?(File.join(directory, Html2rss::Registry::Manifest::MANIFEST_FILE)) + Store.bundle_present_at?(directory) end ## diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 1bd69f28d..760fa28e0 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -49,6 +49,22 @@ def bundle_present?(registry_id) bundle_present_at?(registry_dir(registry_id)) end + ## + # @param path [String] bundle directory + # @return [Boolean] + def bundle_present_at?(path) + File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + + ## + # @param path [String] bundle directory + # @return [Html2rss::Registry::Manifest] + def read_manifest(path) + Html2rss::Registry::Manifest.parse( + File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + end + ## # @param registry_id [String, Symbol] # @return [String] verified staging directory for a registry id @@ -205,23 +221,13 @@ def promote_bundle!(staged_dir, target, backup) FileUtils.rm_rf(backup) end - ## - # @param path [String] - # @return [Boolean] - def bundle_present_at?(path) - File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) - end - ## # @param path [String] # @return [String, nil] def manifest_version_at(path) return nil unless bundle_present_at?(path) - manifest = Html2rss::Registry::Manifest.parse( - File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) - ) - manifest.version + read_manifest(path).version rescue Html2rss::Registry::ManifestError nil end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 9c110e915..ff012e845 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -373,6 +373,7 @@ def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Me Html2rss::Registry::Archive.extract!(io, into: staged_dir) end + # Verify before promote: network-trust seam; Index reload still verifies on disk via Bundle.load. Html2rss::Registry::Verifier.verify!( staged_dir, trust: :signed, @@ -421,9 +422,7 @@ def enforce_max_version!(entry, staged_dir) # @param staged_dir [String] # @return [Html2rss::Registry::Manifest] def read_manifest!(staged_dir) - Html2rss::Registry::Manifest.parse( - File.read(File.join(staged_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) - ) + Store.read_manifest(staged_dir) end ## diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 839d9e13a..91f8c424e 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'fileutils' -require 'openssl' require 'stringio' require 'yaml' @@ -29,7 +28,7 @@ def build_signed_tarball manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) - sign_manifest!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) + Html2rss::Registry::TestSupport.sign!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) pack_bundle_dir(bundle_dir) ensure @@ -64,12 +63,6 @@ def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop } ) end - - def sign_manifest!(manifest, key_pem:, bundle_dir:) - private_key = OpenSSL::PKey.read(key_pem) - signature = private_key.sign(nil, manifest.canonical_bytes) - File.write(File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), [signature].pack('m0')) - end end RSpec.configure do |config| From 14de8e482ab138aa8fab1b87f3a6c87a52c0a1f0 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:51:48 +0200 Subject: [PATCH 12/19] chore(registry): dedup fixtures and sync docs --- AGENTS.md | 2 +- CONTEXT.md | 9 +++++ .../sync/bundle/configs/phys.org/weekly.yml | 25 -------------- .../support.apple.com/en_gb_ht201222.yml | 34 ------------------- spec/support/registry.rb | 24 ------------- spec/support/registry_sync.rb | 30 ++++++++++++++-- 6 files changed, 38 insertions(+), 86 deletions(-) delete mode 100644 spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml delete mode 100644 spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml delete mode 100644 spec/support/registry.rb diff --git a/AGENTS.md b/AGENTS.md index 80d1bdb08..d7cd47803 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,7 +64,7 @@ Public feed-directory metadata from verified registry bundles and local `feeds.y | Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | | Disabled response | `404` with `{ "error": "catalog_disabled" }` | | Registry entries | `Registry::Index.current.catalog_rows` — loads signed bundles from `config/registries.yml`; adds `source: registry`, `registry: ` | -| Local entries | `Registry::LocalCatalog` includes `feeds.yml` feeds only when `directory.title` is set (`source: local`) | +| Local entries | `Registry::Index` catalog rows include `feeds.yml` feeds only when `directory.title` is set (`source: local`) | | Per-registry privacy | `catalog: false` in `registries.yml` omits that registry from the API (feeds still served) | | Starter feeds (UI) | Frontend `selectStarterFeeds` when feed creation is disabled; catalog find uses full catalog when enabled | | Catalog find | `findCatalogEntries` → multi-hit list under create URL; links via `catalogFeedHref` (path + defaults) | diff --git a/CONTEXT.md b/CONTEXT.md index 81c2ec430..9c0f8b1d6 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -54,3 +54,12 @@ Audit channel: snake_case `security_event` with IP / user-agent / token hash. Au ### LogEvent Shared emit plumbing for both channels (`RequestContext`, `LogSanitizer`, `AppLogger` / Sentry). Not a third public facade. + +### Registry Index +Backend merge owner for registry bundles and local `feeds.yml` feeds. Builds catalog wire rows (`Registry::Index::CatalogRow`), enforces load-time trust and channel-domain allowlists, and serves `config_for` / `catalog_rows` / `status`. + +### Registry Sync +Backend orchestration for fetch → verify → stage/promote of signed registry bundles. Owns boot seeding, background refresh, CLI exit codes, and catalog-change telemetry after promotion. + +### Sync Transport +Backend HTTPS fetch, sync URL resolution (GitHub releases and channel defaults), and manifest version gating used by `Registry::Sync` and parse-time config resolution. diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml deleted file mode 100644 index 4c0102d03..000000000 --- a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml +++ /dev/null @@ -1,25 +0,0 @@ -registry: - id: phys.org/weekly -directory: - topics: - - science - title: "Phys.org — Weekly" - summary: "Top science news of the week from Phys.org." -channel: - language: en - title: "Phys.org — Weekly" - url: https://phys.org/weekly-news/ - time_zone: Europe/London - ttl: 1440 -selectors: - items: - selector: ".sorted-news-list .sorted-article-content" - title: - selector: "h4" - category: - selector: ".text-info" - categories: - - category - url: - selector: ".news-link" - extractor: "href" diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml deleted file mode 100644 index 83c1c81b6..000000000 --- a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml +++ /dev/null @@ -1,34 +0,0 @@ -registry: - id: support.apple.com/en_gb_ht201222 -directory: - topics: - - tech - - security - title: "Apple Support — Security releases" - summary: "Apple security update and release notes (HT201222 / related)." -strategy: botasaurus -channel: - title: "Apple Support — Security releases" - url: https://support.apple.com/en-gb/100100 - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_for_selector: ".table-wrapper table tbody tr a" - wait_timeout_seconds: 20 -selectors: - items: - selector: ".table-wrapper table tbody > tr:not(:first-child)" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - description: - selector: "td:nth-child(2)" - published_at: - selector: "td:nth-child(3)" - post_process: - - name: parse_time diff --git a/spec/support/registry.rb b/spec/support/registry.rb deleted file mode 100644 index c07e52949..000000000 --- a/spec/support/registry.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module RegistryTestHelpers - FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) - DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') - - module_function - - def configure_registry_fixtures! - ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG - ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') - end - - def reset_registry! - Html2rss::Web::Registry::Index.reload! - end -end - -RSpec.configure do |config| - config.before do - RegistryTestHelpers.configure_registry_fixtures! - RegistryTestHelpers.reset_registry! - end -end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 91f8c424e..398a24865 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -4,11 +4,28 @@ require 'stringio' require 'yaml' +module RegistryTestHelpers + FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) + DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') + + module_function + + def configure_registry_fixtures! + ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') + end + + def reset_registry! + Html2rss::Web::Registry::Index.reload! + end +end + module RegistrySyncTestHelpers FIXTURE_KEYS_ROOT = File.expand_path('../fixtures/registries/keys', __dir__) TEST_PUBLIC_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pub')) TEST_PRIVATE_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pem')) SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) + OFFICIAL_FIXTURES_ROOT = File.join(RegistryTestHelpers::FIXTURES_ROOT, 'official') SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') @@ -22,9 +39,13 @@ def configure_sync_registry! Html2rss::Web::Registry::Index.reload! end - def build_signed_tarball + def build_signed_tarball # rubocop:disable Metrics/MethodLength bundle_dir = Dir.mktmpdir('signed-registry-bundle') - FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) + FileUtils.cp_r(File.join(OFFICIAL_FIXTURES_ROOT, 'configs'), File.join(bundle_dir, 'configs')) + FileUtils.cp( + File.join(SYNC_FIXTURES_ROOT, 'bundle', Html2rss::Registry::Manifest::MANIFEST_FILE), + File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE) + ) manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) @@ -66,6 +87,11 @@ def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop end RSpec.configure do |config| + config.before do + RegistryTestHelpers.configure_registry_fixtures! + RegistryTestHelpers.reset_registry! + end + config.before do |example| next unless example.metadata[:registry_sync] From 94fff3bea05ffe9749c2479a461b3ebaeaf04285 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:42 +0200 Subject: [PATCH 13/19] refactor(ruby4): apply leading operators and it blocks on registry branch Enable Style/ItBlockParameter, normalize empty-string predicates, and adopt it blocks across registry config and transport helpers. --- .rubocop.yml | 4 ++++ app/web/registry/config.rb | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index ee5e2c04c..767dc7182 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -38,6 +38,10 @@ Style/Documentation: AllowedConstants: - App +Style/ItBlockParameter: + Enabled: true + EnforcedStyle: allow_single_line + RSpec/SpecFilePathFormat: Exclude: - 'spec/html2rss/web/app/*_spec.rb' diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 789bd07d6..468d5f11c 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -91,7 +91,7 @@ def parse_snapshot precedence = Array(document[:precedence]).map(&:to_s) precedence = DEFAULT_PRECEDENCE if precedence.empty? - entries = precedence.to_h { |registry_id| [registry_id, parse_entry(registry_id, document)] } + entries = precedence.to_h { [it, parse_entry(it, document)] } missing = precedence - entries.keys raise Errors::ConfigError, "Missing registry definitions: #{missing.join(', ')}" unless missing.empty? @@ -166,7 +166,7 @@ def parse_entry(registry_id, document) def build_entry(registry_id, raw) sync = sync_section(raw) case raw - in { path: path } if path&.then { |value| !value.empty? } + in { path: path } if path&.then { !it.empty? } entry_attributes(registry_id, raw, :path, expand_path(path.to_s), nil, nil) else build_sync_entry(registry_id, raw, sync) @@ -192,7 +192,7 @@ def build_sync_entry(registry_id, raw, sync) # @param sync_channel [String, nil] # @return [String] def resolved_sync_url(sync_url, sync_channel) - return sync_url if sync_url && !sync_url.empty? + return sync_url unless sync_url.to_s.empty? SyncTransport.resolve_channel_url(sync_channel) end @@ -259,14 +259,14 @@ def optional_string(value) # @param value [Object] # @return [Array] def parse_allowed_channel_domains(value) - Array(value).map { |domain| domain.to_s.strip }.reject(&:empty?) + Array(value).map { it.to_s.strip }.reject(&:empty?) end ## # @param public_key_id [String, nil] # @return [String] def public_key_id_for(public_key_id) - return public_key_id if public_key_id && !public_key_id.empty? + return public_key_id unless public_key_id.to_s.empty? 'html2rss:registry:2026' end From 558b6bdfc408574666e937db125ed72e9e0a2b4e Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:55 +0200 Subject: [PATCH 14/19] refactor(registry): dedupe transport, status lookup, and deep_dup Promote StructuredData.deep_dup for shared cloning, add Store.manifest_mtime, and centralize registry status lookup helpers. --- app/web/config/config_snapshot.rb | 33 ++----------------------------- app/web/config/local_config.rb | 21 +++----------------- app/web/registry/store.rb | 12 +++++++++++ app/web/structured_data.rb | 29 +++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 49 deletions(-) create mode 100644 app/web/structured_data.rb diff --git a/app/web/config/config_snapshot.rb b/app/web/config/config_snapshot.rb index 0a8fc03c5..0afa29fdd 100644 --- a/app/web/config/config_snapshot.rb +++ b/app/web/config/config_snapshot.rb @@ -47,7 +47,7 @@ def normalize_feeds(raw_feeds) return {} unless raw_feeds.is_a?(Hash) raw_feeds.each_with_object({}) do |(name, config), memo| - memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: deep_dup(config).freeze) + memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: StructuredData.deep_dup(config).freeze) end end @@ -68,7 +68,7 @@ def normalize_accounts(raw_accounts) # @param accounts [Array] # @return [Hash{Symbol=>Object}] def normalized_global_hash(global_hash, accounts) - normalized = deep_dup(global_hash) + normalized = StructuredData.deep_dup(global_hash) return normalized unless normalized.key?(:auth) normalized[:auth] = normalized_auth_hash(normalized[:auth], accounts) @@ -85,35 +85,6 @@ def normalized_auth_hash(auth_hash, accounts) end auth end - - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - deep_dup_hash(value) - when Array - deep_dup_array(value) - when String - value.dup - else - value - end - end - - # @param value [Hash] - # @return [Hash] - def deep_dup_hash(value) - value.each_with_object({}) do |(key, val), memo| - memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) - end - end - - # @param value [Array] - # @return [Array] - def deep_dup_array(value) - value.map { |element| deep_dup(element) } - end end end end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index 604643d00..efaac838a 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -41,13 +41,13 @@ def find(name) ## # @return [Hash] def feeds - snapshot.feeds.transform_values { |feed| deep_dup(feed.raw) } + snapshot.feeds.transform_values { StructuredData.deep_dup(it.raw) } end ## # @return [Hash] def global - deep_dup(snapshot.global) + StructuredData.deep_dup(snapshot.global) end ## @@ -104,7 +104,7 @@ def local_feed_config(normalized_name) config = snapshot.feeds[normalized_name.to_sym] return nil unless config - deep_dup(config.raw) + StructuredData.deep_dup(config.raw) end # @param normalized_name [String] @@ -118,21 +118,6 @@ def registry_feed_config(normalized_name) def normalize_name(name) name.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') end - - # Deep-duplicates nested config structures to avoid mutating shared data. - # - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - value.transform_values { |val| deep_dup(val) } - when Array - value.map { |element| deep_dup(element) } - else - value - end - end end end end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 760fa28e0..5deedfca1 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -186,6 +186,18 @@ def write_sync_state!(registry_id, last_error:, last_sync_at: Time.now.utc) File.write(sync_state_path, JSON.generate(state)) end + ## + # @param path [String, nil] + # @return [Time, nil] + def manifest_mtime(path) + return nil unless path && File.directory?(path) + + manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + return nil unless File.file?(manifest_path) + + File.mtime(manifest_path) + end + private ## diff --git a/app/web/structured_data.rb b/app/web/structured_data.rb new file mode 100644 index 000000000..f4278c419 --- /dev/null +++ b/app/web/structured_data.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Html2rss + module Web + ## + # Shared helpers for cloning nested Hash/Array config structures. + module StructuredData + module_function + + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) + case value + when Hash + value.each_with_object({}) do |(key, val), memo| + memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) + end + when Array + value.map { deep_dup(it) } + when String + value.dup + else + value + end + end + end + end +end From 5634a2f7461c6623ae957e0600db5dc7a76d5742 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 15/19] perf(registry): use Set for catalog diff and tighten hot paths Unify official GitHub asset resolution, merge catalog rows with to_h, replace status.find duplication with status_entry_for, and memoize extra_allowed_hosts. --- app/web/registry/index.rb | 51 +++++++++++---------------- app/web/registry/sync.rb | 33 +++++++++--------- app/web/registry/sync_transport.rb | 56 ++++++++++++++---------------- 3 files changed, 63 insertions(+), 77 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index edea52fa4..ed3c4f91d 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Sole merge owner for registry bundles and local {LocalConfig} feeds. - class Index # rubocop:disable Metrics/ClassLength + class Index RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) @@ -96,7 +96,7 @@ def config_for(feed_id) loaded_bundles.each_value do |bundle| config = bundle.configs[normalized_id] - return deep_dup(config) if config + return StructuredData.deep_dup(config) if config end nil @@ -105,9 +105,11 @@ def config_for(feed_id) ## # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows - rows = registry_catalog_rows - local_catalog_rows.each { |row| rows[row.id] = row } - rows.values.sort_by(&:id).map(&:to_h) + registry_catalog_rows + .merge(local_catalog_rows.to_h { [it.id, it] }) + .values + .sort_by(&:id) + .map(&:to_h) end ## @@ -132,6 +134,13 @@ def status end end + ## + # @param registry_id [String, Symbol] + # @return [StatusEntry, nil] + def status_entry_for(registry_id) + status.find { it.id == registry_id.to_s } + end + private ## @@ -160,7 +169,7 @@ def local_catalog_rows ## # @return [Hash{String => RegistryBundle}] def loaded_bundles - @loaded_bundles ||= Config.precedence.to_h { |registry_id| [registry_id, load_bundle(registry_id)] }.compact + @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact end ## @@ -215,14 +224,14 @@ def signed_bundle?(bundle_dir, entry) # @param bundle [RegistryBundle] # @return [void] # @raise [Errors::LoadError] when a config channel URL violates the allowlist - def enforce_scrape_policy!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + def enforce_scrape_policy!(entry, bundle) allowed = entry.allowed_channel_domains return if allowed.nil? || allowed.empty? bundle.configs.each do |feed_id, config| channel_url = config.dig(:channel, :url) host = channel_host_for(channel_url) - next if host && allowed.any? { |domain| channel_domain_allowed?(host, domain) } + next if host && allowed.any? { channel_domain_allowed?(host, it) } raise Errors::LoadError, "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ @@ -291,13 +300,7 @@ def normalize_feed_id(feed_id) # @param registry_id [String] # @return [Time, nil] def bundle_updated_at(registry_id) - path = bundle_directory(Config.entry(registry_id)) - return nil unless path && File.directory?(path) - - manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) - return nil unless File.file?(manifest_path) - - File.mtime(manifest_path) + Store.manifest_mtime(bundle_directory(Config.entry(registry_id))) end ## @@ -307,25 +310,11 @@ def local_config_for(feed_id) feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] return nil unless feed - deep_dup(feed) + StructuredData.deep_dup(feed) rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound nil end - - ## - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - value.transform_values { |entry| deep_dup(entry) } - when Array - value.map { |entry| deep_dup(entry) } - else - value - end - end - end # rubocop:enable Metrics/ClassLength + end end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index ff012e845..6a37a635c 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Fetches, verifies, and stores signed registry bundles. - module Sync # rubocop:disable Metrics/ModuleLength + module Sync SyncStatus = Data.define( :registry_id, :mode, @@ -23,7 +23,7 @@ module Sync # rubocop:disable Metrics/ModuleLength @timer_started = false REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - class << self # rubocop:disable Metrics/ClassLength + class << self ## # Resolves the download URL for a sync-mode registry id. # @@ -60,8 +60,8 @@ def promote_staged!(registry_id:) # @return [Array] def status(registry_id: nil) rows = Index.current.status - rows = rows.select { |row| row.id == registry_id.to_s } if registry_id - rows.map { |row| sync_status_for(row) } + rows = rows.select { it.id == registry_id.to_s } if registry_id + rows.map { sync_status_for(it) } end ## @@ -142,7 +142,7 @@ def with_registry_lock(registry_id, &) # @param registry_id [String, Symbol] # @param dry_run [Boolean] # @return [SyncStatus] - def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength entry = Config.entry(registry_id) if entry.mode == :path raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" @@ -159,7 +159,7 @@ def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/Meth record_success!(registry_id, promoted: false) end end - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + sync_status_for(Index.current.status_entry_for(registry_id)) rescue StandardError => error record_failure!(registry_id, error) unless dry_run raise @@ -184,15 +184,16 @@ def promote_staged_bundle!(registry_id:) # rubocop:disable Metrics/AbcSize, Metr Index.reload! report_catalog_change!(registry_id, previous_bundle) record_success!(registry_id, promoted: true) + status_row = Index.current.status_entry_for(registry_id) Observability.emit( event_name: 'registry.promote_staged', outcome: 'success', details: { registry_id: registry_id.to_s, - version: Index.current.status.find { |row| row.id == registry_id.to_s }&.version + version: status_row&.version } ) - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + sync_status_for(status_row) end ## @@ -226,11 +227,11 @@ def report_catalog_change!(registry_id, previous_bundle) # @param previous_bundle [Index::RegistryBundle, nil] # @param new_bundle [Index::RegistryBundle] # @return [Hash{Symbol => Object}] - def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + def build_catalog_diff(previous_bundle, new_bundle) previous_version = previous_bundle&.manifest&.version new_version = new_bundle.manifest.version - previous_ids = catalog_ids(previous_bundle) - new_ids = catalog_ids(new_bundle) + previous_ids = Set.new(catalog_ids(previous_bundle)) + new_ids = Set.new(catalog_ids(new_bundle)) added = new_ids - previous_ids removed = previous_ids - new_ids @@ -239,8 +240,8 @@ def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/Me { previous_version:, version: new_version, - added:, - removed: + added: added.to_a, + removed: removed.to_a } end @@ -431,7 +432,7 @@ def read_manifest!(staged_dir) # @return [void] def record_success!(registry_id, promoted:) # rubocop:disable Metrics/MethodLength Store.write_sync_state!(registry_id, last_error: nil) - row = Index.current.status.find { |entry| entry.id == registry_id.to_s } + row = Index.current.status_entry_for(registry_id) Observability.emit( event_name: 'registry.sync', outcome: 'success', @@ -474,8 +475,8 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength last_error: state.last_error ) end - end # rubocop:enable Metrics/ClassLength - end # rubocop:enable Metrics/ModuleLength + end + end end end end diff --git a/app/web/registry/sync_transport.rb b/app/web/registry/sync_transport.rb index d67461a59..52282604f 100644 --- a/app/web/registry/sync_transport.rb +++ b/app/web/registry/sync_transport.rb @@ -9,7 +9,7 @@ module Web module Registry ## # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. - module SyncTransport # rubocop:disable Metrics/ModuleLength + module SyncTransport DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -56,13 +56,16 @@ def fetch!(url, policy: RedirectPolicy.default) # @param entry [Entry] # @return [String] def resolve(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + return entry.sync_url unless entry.sync_url.to_s.empty? pin_version = entry.sync_policy.pin_version if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + unless pin_version.to_s.empty? + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag: pin_version) + return resolve_official_asset_url(api_url, tag: pin_version) + end - return resolve_official_download_url + return resolve_official_asset_url(OFFICIAL_GITHUB_RELEASES_API) end raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" @@ -72,7 +75,7 @@ def resolve(entry) # @param sync_channel [String, nil] # @return [String] def resolve_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + channel = sync_channel.to_s.empty? ? Config::DEFAULT_OFFICIAL_SYNC_CHANNEL : sync_channel return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL raise Errors::ConfigError, "Unknown sync channel '#{channel}'" @@ -119,24 +122,31 @@ def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/Me ## # @return [Array] def extra_allowed_hosts - ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - .split(',') - .map(&:strip) - .reject(&:empty?) + raw = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + if defined?(@extra_allowed_hosts_raw) && @extra_allowed_hosts_raw == raw + return @extra_allowed_hosts + end + + @extra_allowed_hosts_raw = raw + @extra_allowed_hosts = raw.split(',').map(&:strip).reject(&:empty?).freeze end ## - # @param tag [String] + # @param api_url [String] + # @param tag [String, nil] # @return [String] - def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength - api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + def resolve_official_asset_url(api_url, tag: nil) response_body = fetch!(api_url) release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } url = asset&.dig(:browser_download_url) unless url - raise Errors::SyncError, - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + message = if tag + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + else + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" + end + raise Errors::SyncError, message end url @@ -144,20 +154,6 @@ def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLengt raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" end - ## - # @return [String] - def resolve_official_download_url - response_body = fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - ## # @param version [String] # @return [String] @@ -237,7 +233,7 @@ def read_body!(response) body end - end # rubocop:enable Metrics/ModuleLength + end end end end From e8e61be183be6db49991e57117abb526b0f2d488 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 16/19] test(registry): drop send pins and table-drive transport specs Cover channel allowlist via integration config load and table-drive SyncTransport version gate examples; adopt it blocks in registry specs. --- spec/html2rss/web/registry/index_spec.rb | 77 ++++++++---------------- spec/html2rss/web/registry/sync_spec.rb | 20 +++--- 2 files changed, 35 insertions(+), 62 deletions(-) diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index f5797ece8..f6985afce 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -27,7 +27,7 @@ describe '#catalog_rows' do it 'includes registry rows with source and registry fields' do rows = described_class.current.catalog_rows - phys = rows.find { |row| row.fetch(:id) == 'phys.org/weekly' } + phys = rows.find { it.fetch(:id) == 'phys.org/weekly' } expect(phys).to include( source: 'registry', @@ -39,12 +39,12 @@ it 'omits catalog-disabled registries from the catalog API rows' do rows = described_class.current.catalog_rows - expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') + expect(rows.map { it.fetch(:id) }).not_to include('secret.example/private') end it 'prefers the first registry in precedence for duplicate feed ids' do rows = described_class.current.catalog_rows - apple = rows.find { |row| row.fetch(:id) == 'support.apple.com/en_gb_ht201222' } + apple = rows.find { it.fetch(:id) == 'support.apple.com/en_gb_ht201222' } expect(apple.fetch(:registry)).to eq('official') end @@ -58,7 +58,7 @@ ) rows = described_class.current.catalog_rows - local = rows.find { |row| row.fetch(:id) == 'team/releases' } + local = rows.find { it.fetch(:id) == 'team/releases' } expect(local).to include( source: 'local', @@ -73,7 +73,7 @@ describe '#status' do it 'reports loaded registry metadata' do status = described_class.current.status - official = status.find { |entry| entry.id == 'official' } + official = status.find { it.id == 'official' } expect(official).to have_attributes( version: 'test-fixture', @@ -82,52 +82,6 @@ end end - describe 'channel domain allowlist' do - let(:index) { described_class.new } - let(:entry) do - Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :path, - path: nil, - sync_channel: nil, - sync_url: nil, - catalog: true, - public_key_id: 'test', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), - allowed_channel_domains: ['phys.org'] - ) - end - let(:allowed_bundle) do - described_class::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } - }, - catalog_entries: [] - ) - end - - it 'allows suffix-matching channel domains' do - expect { index.send(:enforce_scrape_policy!, entry, allowed_bundle) }.not_to raise_error - end - - it 'raises when a config channel domain is not allowlisted' do - blocked_bundle = described_class::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } - }, - catalog_entries: [] - ) - - expect { index.send(:enforce_scrape_policy!, entry, blocked_bundle) } - .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) - end - end - describe 'allowed_channel_domains' do let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } @@ -135,6 +89,27 @@ FileUtils.rm_f(config_path) end + it 'allows suffix-matching channel domains via config load' do # rubocop:disable RSpec/ExampleLength + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + path: spec/fixtures/registries/official + catalog: true + allowed_channel_domains: + - phys.org + - apple.com + YAML + ENV['REGISTRIES_CONFIG'] = config_path + described_class.reload! + + expect(described_class.current.config_for('phys.org/weekly')).to include( + channel: hash_including(url: 'https://phys.org/weekly-news/') + ) + end + it 'rejects bundles with channel URLs outside the allowlist' do # rubocop:disable RSpec/ExampleLength FileUtils.mkdir_p(File.dirname(config_path)) File.write(config_path, <<~YAML) diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 20245c131..1440d0cde 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -219,17 +219,15 @@ def write_policy_config(yaml) describe Html2rss::Web::Registry::SyncTransport do describe '.exceeds_max?' do - it 'compares dotted numeric versions' do - expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) - expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) - end - - it 'strips a leading v prefix before comparing' do - expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) - end - - it 'returns false when max_version is unset' do - expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + [ + ['2026.08.22', '2026.08.21', true], + ['2026.08.21', '2026.08.22', false], + ['v2026.08.22', '2026.08.21', true], + ['2026.08.22', nil, false] + ].each do |manifest_version, max_version, expected| + it "returns #{expected} for #{manifest_version} vs #{max_version.inspect}" do + expect(described_class.exceeds_max?(manifest_version, max_version)).to be(expected) + end end end From f337b47c7aa83a16b4280b1ced949cb7aeac333f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 17/19] refactor(api): modernize configs and root_metadata helpers Skipped: endless-method rubocop constraints removed net LOC win on configs.rb; root_metadata unchanged. From f38b7a7e812a759c0238c6420be9dbb2f4194f8f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 18:01:17 +0200 Subject: [PATCH 18/19] fix(registry): restore rubocop disables and constant-cache memoization Use EXTRA_ALLOWED_HOSTS_CACHE instead of module ivars so thread-safety cops pass while keeping per-process host memoization for registry sync. --- app/web/registry/index.rb | 4 ++-- app/web/registry/sync.rb | 10 +++++----- app/web/registry/sync_transport.rb | 14 ++++++-------- app/web/structured_data.rb | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index ed3c4f91d..7159940e6 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Sole merge owner for registry bundles and local {LocalConfig} feeds. - class Index + class Index # rubocop:disable Metrics/ClassLength RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) @@ -314,7 +314,7 @@ def local_config_for(feed_id) rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound nil end - end + end # rubocop:enable Metrics/ClassLength end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 6a37a635c..b489f1fe5 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Fetches, verifies, and stores signed registry bundles. - module Sync + module Sync # rubocop:disable Metrics/ModuleLength SyncStatus = Data.define( :registry_id, :mode, @@ -23,7 +23,7 @@ module Sync @timer_started = false REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - class << self + class << self # rubocop:disable Metrics/ClassLength ## # Resolves the download URL for a sync-mode registry id. # @@ -227,7 +227,7 @@ def report_catalog_change!(registry_id, previous_bundle) # @param previous_bundle [Index::RegistryBundle, nil] # @param new_bundle [Index::RegistryBundle] # @return [Hash{Symbol => Object}] - def build_catalog_diff(previous_bundle, new_bundle) + def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength previous_version = previous_bundle&.manifest&.version new_version = new_bundle.manifest.version previous_ids = Set.new(catalog_ids(previous_bundle)) @@ -475,8 +475,8 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength last_error: state.last_error ) end - end - end + end # rubocop:enable Metrics/ClassLength + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/registry/sync_transport.rb b/app/web/registry/sync_transport.rb index 52282604f..220ef1902 100644 --- a/app/web/registry/sync_transport.rb +++ b/app/web/registry/sync_transport.rb @@ -9,7 +9,7 @@ module Web module Registry ## # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. - module SyncTransport + module SyncTransport # rubocop:disable Metrics/ModuleLength DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -26,6 +26,7 @@ module SyncTransport OFFICIAL_GITHUB_TAG_RELEASES_API = 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + EXTRA_ALLOWED_HOSTS_CACHE = {} # rubocop:disable Style/MutableConstant RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do ## @@ -123,19 +124,16 @@ def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/Me # @return [Array] def extra_allowed_hosts raw = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - if defined?(@extra_allowed_hosts_raw) && @extra_allowed_hosts_raw == raw - return @extra_allowed_hosts + EXTRA_ALLOWED_HOSTS_CACHE.fetch(raw) do + EXTRA_ALLOWED_HOSTS_CACHE[raw] = raw.split(',').map(&:strip).reject(&:empty?).freeze end - - @extra_allowed_hosts_raw = raw - @extra_allowed_hosts = raw.split(',').map(&:strip).reject(&:empty?).freeze end ## # @param api_url [String] # @param tag [String, nil] # @return [String] - def resolve_official_asset_url(api_url, tag: nil) + def resolve_official_asset_url(api_url, tag: nil) # rubocop:disable Metrics/MethodLength response_body = fetch!(api_url) release = JSON.parse(response_body, symbolize_names: true) asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } @@ -233,7 +231,7 @@ def read_body!(response) body end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/structured_data.rb b/app/web/structured_data.rb index f4278c419..d829d6d5c 100644 --- a/app/web/structured_data.rb +++ b/app/web/structured_data.rb @@ -10,7 +10,7 @@ module StructuredData ## # @param value [Object] # @return [Object] - def deep_dup(value) + def deep_dup(value) # rubocop:disable Metrics/MethodLength case value when Hash value.each_with_object({}) do |(key, val), memo| From 9f1ebaa11a5739485b7ace7ca7dcce46952ab6f0 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 18:01:20 +0200 Subject: [PATCH 19/19] docs: codify Ruby 4.0 style for agents and contributors --- .rubocop.yml | 1 + AGENTS.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/README.md | 5 ++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 767dc7182..328acf9b4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,6 +5,7 @@ plugins: - rubocop-thread_safety AllCops: + TargetRubyVersion: 4.0 DisplayCopNames: true NewCops: enable Exclude: diff --git a/AGENTS.md b/AGENTS.md index d7cd47803..90cd9d06f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,45 @@ See [docs/design-system.md](docs/design-system.md) for visual rules. - **No host execution:** All commands MUST run inside the Dev Container via `make` or `bundle exec`. - **No skipped quality gate:** Opening a PR without a green Dev Container gate is forbidden. If the gate cannot run, do not open the PR; fix the environment or hand off with explicit blocker + next command for the user. +## Ruby 4 Style + +**Ruby 4.0+ only** (see `.tool-versions`). No Ruby 3.x backward-compat shims, guards, or dual-path APIs. + +### Baseline + +- `# frozen_string_literal: true` on every `.rb` file +- Plain Ruby — no ActiveSupport +- Keyword arguments for public multi-arg APIs +- Typed YARD on public methods in `app/` (`@param`, `@return`) — enforced by `make yard-verify-public-docs` + +### Modern syntax (prefer consistently) + +| Idiom | Use instead of | +| --- | --- | +| Leading `&&` / `\|\|` at line start (Ruby 4) | Trailing operators on long wrapped conditions | +| `it` in single-parameter blocks | `{ \|x\| x.foo }` when block has one arg only | +| Pattern matching (`in`, `case … in`) | Deep `if/elsif` chains on shape | +| `Data.define` | OpenStruct / hand-rolled structs | +| `filter_map`, `index_by`, `then`, `match?` | Verbose `map`/`compact`, nested `if`, `=~` | +| Endless `def` | One-line pure helpers when RuboCop allows | +| Core `Set` (no `require 'set'`) | Array membership/diff on growing collections | + +### Performance (agent defaults) + +- **Set** for catalog/diff/membership when sizes can grow +- **Memoize** repeated `ENV.fetch` / pure computations on hot paths +- **One owner** for duplicated helpers — dedupe before splitting into new files +- **Functional iterators** over imperative loops +- **No metric-driven micro-methods** whose only purpose is satisfying RuboCop metrics +- Do **not** document ZJIT/Ruby Box/Ractor as defaults + +### Web-specific deltas + +- Prefer `class << self` + `private` over `module_function` (see docs/README Architectural Constraints) +- Do not use `send(...)` to reach private APIs in app code or specs +- Specs: table-drive matrices; `:aggregate_failures` for discriminating multi-assert examples +- LOC: dedupe/unify before extracting — new files only when they buy a real seam or test surface + ## Config catalog API Public feed-directory metadata from verified registry bundles and local `feeds.yml` entries. diff --git a/docs/README.md b/docs/README.md index 7c27ebb33..653820337 100644 --- a/docs/README.md +++ b/docs/README.md @@ -167,12 +167,15 @@ Search these pages for examples, plugins, and configuration options: ## Architectural Constraints - **No Persistence**: Do not add databases, ORMs, or background job systems. -- **Backend Style**: +- **Backend Style** (Ruby **4.0+** only — see [AGENTS.md](../AGENTS.md#ruby-4-style)): - Keep the main `app.rb` thin; organize routes in `Html2rss::Web::Routes::*`. - For helpers, use `class << self` and `private` methods. Avoid `module_function`. - Use YARD doc comments for all public methods in `app/`. - Add `# frozen_string_literal: true` to all Ruby files. - Do not use `send(...)` to reach into private APIs; expose what is needed at the module level. + - Prefer leading `&&` / `||` at line start for wrapped conditions; `it` in single-parameter blocks; pattern matching over deep `if/elsif` chains. + - Prefer `Data.define`, `filter_map`, `index_by`, `then`, `match?`, and core `Set` (no `require 'set'`) over OpenStruct, verbose `map`/`compact`, nested `if`, `=~`, and array membership on growing collections. + - Dedupe helpers before extracting new files; use `Set` and memoization on hot paths; table-drive specs with `:aggregate_failures` for multi-assert outcomes. - **Frontend Style**: - Follow visual and CSS rules in [design-system.md](design-system.md). - Use Preact components in `frontend/src/`.