diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0b0cd0..3533ebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,13 @@ name: CI # # No credential is configured here and nothing reaches an external service: the # suite must need neither (doc/POLICY.md Invariant 6). Examples tagged :network -# are excluded by default, and the Gemfile's optional :plugins group is not +# are excluded by default, and none of the Gemfile's optional groups is # installed, so no plugin's own gem is a condition of this workflow passing. +# +# This is the required check, and it is deliberately the minimal configuration: +# what it proves on every commit is that the framework needs nothing but its +# four runtime dependencies. The all-plugins configuration is a separate, +# non-required workflow, plugins.yml. on: push: diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml new file mode 100644 index 0000000..ad9b2d7 --- /dev/null +++ b/.github/workflows/plugins.yml @@ -0,0 +1,55 @@ +name: Optional plugin dependencies + +# Installs the Gemfile's optional `plugins` group and runs the same suite, so +# that the second documented way of setting up a checkout is checked as well as +# described: the group resolves, and the plugin specs it brings in pass. +# +# This is not the required check. ci.yml is, and it installs none of this: what +# has to hold on every commit is that the framework runs on its own runtime +# dependencies (doc/POLICY.md sections 9.1 and 11). This workflow is the other +# half of that statement, and it is held to the same standard — a failure here +# is a real failure, to be fixed rather than silenced. +# +# One Ruby version, not the matrix. This checks the dependency configuration, +# which the required workflow already checks across versions without it. +# +# Still no credential and no external service: the optional gems of the plugins +# that need a running service are in their own groups, outside `plugins`, and +# are not installed here. + +on: + push: + branches: ['**'] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + plugins: + name: All supported optional plugin dependencies + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + + - name: Select the optional plugin group + run: bundle config set --local with plugins + + - name: Install the bundle, including the optional plugin group + run: bundle install --jobs 4 + + - name: Check that the optional gems are in the bundle + run: | + bundle exec ruby -e "require 'nokogiri'; require 'active_record'; \ + require 'sqlite3'; require 'sanitize'; require 'feedbag'; \ + puts 'optional plugin dependencies present'" + + - name: Run the test suite + run: bundle exec rake spec diff --git a/Gemfile b/Gemfile index cc077a8..1f5a83b 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,10 @@ # The runtime and development dependencies are declared in automatic.gemspec, # which this file evaluates. Only the optional, plugin-specific gems are listed # here. See doc/POLICY.md section 9. +# +# `bundle install` with no configuration installs the framework's four runtime +# dependencies and the development ones, and nothing below: a checkout is set +# up to run and to test the framework, not to run every plugin. source 'https://rubygems.org' @@ -11,23 +15,72 @@ gemspec # runtime dependencies of the gem: installing automatic does not install them, # and a Recipe that does not use the plugin does not need them. # -# The group is optional, so `bundle install` does not install it and neither -# the default test suite nor CI depends on it. Install it deliberately, and the -# specs of the plugins that need it then run as part of the ordinary suite: +# Every group here is optional, so nothing below is installed by default and +# neither the default test suite nor required CI depends on any of it. Each gem +# is in two groups: `plugins`, which is all of them at once, and one named +# after what it is for, which is one of them on its own. Both are Bundler +# groups and both are selected the same way, in the checkout's own .bundle +# directory, which is not committed: +# +# bundle config set --local with plugins # all of the below +# bundle config set --local with store # activerecord and sqlite3 only +# bundle config set --local with "store html" +# bundle install # -# BUNDLE_WITH=plugins bundle install -# bundle exec rake +# Setting it in the configuration rather than passing BUNDLE_WITH to one +# command is what makes the gems visible to `bundle exec` afterwards, so the +# specs of the plugins that need them then run as part of the ordinary suite. +# `bundle config unset --local with` returns the checkout to the minimum. # # The table of which plugin needs which gem, and which of those plugins still # work, is in doc/DEPLOYMENT.md and doc/PLUGINS.md section 6. -group :plugins, optional: true do - gem 'nkf' # FilterDescriptionLink - gem 'sanitize' # FilterSanitize - # PublishAmazonS3 and the s3n:// path of StoreFile call AWS::S3, which only - # AWS SDK for Ruby v1 provided. No currently published gem satisfies them, so - # there is nothing to uncomment; they need rework. See doc/PLUGINS.md. - # gem 'dalli' # PublishMemcached - # gem 'fluent-logger' # PublishFluentd and ProvideFluentd - # gem 'xml-simple' # CustomFeedSVNLog +# StorePermalink and StoreFullText, through plugins/store/database.rb. +group :plugins, :store, optional: true do + gem 'activerecord', '>= 7.1', '< 9.0' + gem 'sqlite3', '>= 1.7', '< 3.0' end + +# An HTML parser, for the plugins that read HTML: FilterFullFeed, +# FilterImageSource, FilterDescriptionLink, and FeedParser.parse_html for +# SubscriptionLink and SubscriptionTumblr. PublishMarkdown uses it when it is +# installed and reduces a body to text without it. +group :plugins, :html, optional: true do + gem 'nokogiri', '>= 1.15', '< 2.0' +end + +# FilterSanitize. +group :plugins, :sanitize, optional: true do + gem 'sanitize' +end + +# FilterDescriptionLink, which normalizes a fetched page's encoding with it. +group :plugins, :nkf, optional: true do + gem 'nkf' +end + +# The autodiscovery and inspect subcommands. No plugin and no Recipe uses it. +group :plugins, :autodiscovery, optional: true do + gem 'feedbag', '>= 1.0', '< 2.0' +end + +# The plugins below are Supported (external): each needs a service or a command +# the operator provides, and their specs exercise it rather than a double. They +# are deliberately outside the `plugins` group, so that installing that group +# leaves the suite runnable with nothing else set up. Select one of these by +# its own name when you have what it talks to. +group :memcached, optional: true do + gem 'dalli' # PublishMemcached, with a memcached server +end + +group :fluentd, optional: true do + gem 'fluent-logger' # PublishFluentd and ProvideFluentd, with a Fluentd instance +end + +group :svn_log, optional: true do + gem 'xml-simple' # CustomFeedSVNLog, with the svn command +end + +# PublishAmazonS3 and the s3n:// path of StoreFile call AWS::S3, which only +# AWS SDK for Ruby v1 provided. No currently published gem satisfies them, so +# there is no group to select; they need rework. See doc/PLUGINS.md. diff --git a/README.md b/README.md index 23965b2..a3b748b 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,14 @@ plainly which of its plugins still work. See [`doc/VERSIONS`](doc/VERSIONS). - **Your plugins override the shipped ones.** `~/.automatic/plugins` is searched first, so a shipped plugin can be replaced without touching the installation. - **De-duplication built in.** The store plugins keep a SQLite record of what - has been seen, which is what makes a Recipe safe to run every hour. + has been seen, which is what makes a Recipe safe to run every hour. Their + gems are installed when you use them, not before. - **Retry and interval** on everything that reaches the network, configured per plugin in the Recipe. - **A small installation.** A gem needed by one plugin is not a dependency of - the framework, so installing this does not install an AWS SDK. + the framework: `gem install automatic` brings four pure-Ruby gems and the + command, and installs neither an HTML parser nor a database — let alone an + AWS SDK. - **Honest about what is broken.** Every plugin is classified, with its reason, in [`doc/PLUGINS.md`](doc/PLUGINS.md). Nothing dead is stubbed into looking alive. @@ -163,7 +166,9 @@ The full account is [`doc/BASIC_DESIGN.md`](doc/BASIC_DESIGN.md). - **Ruby 3.3 through 4.0.** CI validates 3.3, 3.4 and 4.0. - A Unix-like system. GNU/Linux and macOS are what it is used on. Windows is not supported. -- A compiler, if `nokogiri` or `sqlite3` build from source on your platform. +- A compiler only if you install an optional plugin gem that builds from source + on your platform, such as `nokogiri` or `sqlite3`. The framework's own + dependencies are pure Ruby. Ruby 3.3 is the floor: it is the oldest maintained release the dependencies are resolved and tested against. Nothing older is tested or supported. @@ -187,33 +192,49 @@ gem install automatic automatic --version ``` +That installs the framework, the command and four pure-Ruby dependencies. +A gem that only one plugin needs is not among them: install it when you use +that plugin, with `gem install nokogiri` or `gem install activerecord sqlite3`. +[`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md) lists which plugin needs which. + ### From a checkout Use a checkout to try the current development version, change the source, -develop a plugin or verify changes before a release: +develop a plugin or verify changes before a release. There are three ways to +set one up; start with the first. ```sh git clone https://github.com/id774/automaticruby.git cd automaticruby + +# Minimal: the framework and its test suite. No optional plugin gem. +bundle install + +# All supported optional plugin dependencies, for plugin work. +bundle config set --local with plugins +bundle install + +# Or start minimal and add one group at a time, as you use its plugins. +bundle config set --local with store bundle install +``` + +```sh bundle exec bin/automatic --version bundle exec rake ``` -`bundle install` resolves the runtime and development dependencies declared by -`Gemfile` and `automatic.gemspec`, and installs the gems needed to run and test -the checkout. If the `bundle` command is unavailable, install Bundler first -with `gem install bundler`. - -In a checkout, every `automatic` below becomes `bundle exec bin/automatic`. -Use `bundle exec rake` to verify the development environment by running the -test suite. +A plain `bundle install` resolves the runtime dependencies declared by +`automatic.gemspec` and the development ones, and installs no optional plugin +gem: those are optional Bundler groups, which are installed only when asked +for. If the `bundle` command is unavailable, install Bundler first with +`gem install bundler`. -The core development setup does not install the optional `plugins` bundle -group. Some plugins need their own gem or external service; install those only -for the plugin being used or developed. Check the plugin catalogue in -[`doc/PLUGINS.md`](doc/PLUGINS.md) and the dependency table in -[`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md). +In a checkout, every `automatic` below becomes `bundle exec bin/automatic`, and +`bundle exec` sees only the bundle — so a plugin's gem is added with a group +rather than with `gem install`. The group names, and which plugin needs which +gem, are in [`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md); what each plugin does is +in [`doc/PLUGINS.md`](doc/PLUGINS.md). ## 6. Quick start @@ -228,11 +249,13 @@ automatic -c ~/.automatic/config/example/feed2markdown.yml `assets/`, and copies the example Recipes into `~/.automatic/config/example`. It never overwrites anything already there. -That Recipe fetches the public Ruby news feed, skips what it has published -before, and appends the rest to `~/.automatic/markdown/feeds.md`. Run it twice: the second -run leaves the file alone, because the store plugin has seen it all. Read the -file, `grep` it, put it in a repository, or hand it to whatever reads text next. -`feed2console.yml` beside it is the same pipeline printing to the terminal. +That Recipe fetches the public Ruby news feed and appends its items to +`~/.automatic/markdown/feeds.md`, using nothing but the framework and what +`gem install automatic` brought. Read the file, `grep` it, put it in a +repository, or hand it to whatever reads text next. `feed2console.yml` beside +it is the same pipeline printing to the terminal. Adding a store plugin, so +that a second run appends only what is new, is step 5 of the Quick Start and +the point at which the first optional gems are installed. To check the framework without any network, write this instead: @@ -485,12 +508,13 @@ COVERAGE=on bundle exec rake spec AUTOMATIC_NETWORK_SPECS=1 bundle exec rake spec ``` -- A plugin whose gem the Gemfile declares in its optional `:plugins` group is - **not verified by the default suite**, because that group is not installed. - Install it to run those specs as part of the ordinary suite: +- A plugin whose gem the Gemfile declares in an optional group is **not + verified by the default suite**, because no optional group is installed. + Install them to run those specs as part of the ordinary suite: ```sh - BUNDLE_WITH=plugins bundle install + bundle config set --local with plugins + bundle install bundle exec rake ``` @@ -502,11 +526,16 @@ COVERAGE=on bundle exec rake spec in CI. Most need a credential, a dead service, or both — read one before running it. -CI installs the bundle, builds the gem, loads the library, runs the CLI and runs -the default suite on each validated Ruby version, from +The required check installs the bundle, builds the gem, loads the library, runs +the CLI and runs the default suite on each validated Ruby version, from [`.github/workflows/ci.yml`](.github/workflows/ci.yml). It configures no secret and installs no optional plugin gem, so no plugin's own dependency is a -condition of a green build. +condition of a change being merged — and what it proves on every commit is that +the framework needs nothing but its own runtime dependencies. A separate, +non-required workflow, +[`.github/workflows/plugins.yml`](.github/workflows/plugins.yml), installs the +`plugins` group and runs the same suite, which is how the all-plugins setup is +checked. ## 13. Development diff --git a/automatic.gemspec b/automatic.gemspec index efb7b4b..62a67ad 100644 --- a/automatic.gemspec +++ b/automatic.gemspec @@ -86,31 +86,26 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.extra_rdoc_files = ['README.md', 'doc/LICENSE.md'] - # Runtime dependencies: what the framework itself needs, plus what the - # documented primary workflow needs — the store plugins, which nearly every - # Recipe uses to avoid repeating its work, and the Markdown publisher. + # Runtime dependencies: what the framework in lib/ requires, and nothing + # else. Requiring `automatic`, loading a Recipe, loading a plugin, running a + # pipeline and the CLI's own work are what these four are for. # - # A gem needed by a single plugin is NOT declared here. It is required inside - # that plugin's own file and installed by the operator who uses the plugin. - # See doc/POLICY.md section 9.1 and doc/DEPLOYMENT.md. + # A gem needed by a plugin is NOT declared here, however useful that plugin + # is. It is required inside the plugin's own file and installed by the + # operator who uses the plugin: `gem install automatic` therefore installs no + # HTML parser, no database and no service client. The optional gems, which + # plugin needs which, and how to install them are in the Gemfile's optional + # groups, doc/DEPLOYMENT.md and doc/POLICY.md section 9.1. # # rexml and rss left the standard library and became gems over the 3.x # series, and nkf followed after 3.3. Each gem listed here is listed because - # something committed here requires it, and a library's move out of the - # standard library is not by itself a reason to declare it: nkf is a plugin's - # dependency and is in the Gemfile's optional group instead. - spec.add_dependency 'activerecord', '>= 7.1', '< 9.0' # store plugins + # a file in lib/ requires it, and a library's move out of the standard + # library is not by itself a reason to declare it: nkf is a plugin's + # dependency and is in the Gemfile's optional groups instead. spec.add_dependency 'activesupport', '>= 7.1', '< 9.0' # plugin loader, XML subscription - spec.add_dependency 'feedbag', '>= 1.0', '< 2.0' # autodiscovery subcommand spec.add_dependency 'hashie', '>= 4.0', '< 6.0' # Recipe - # Used by no framework file on the way in: requiring `automatic` loads no - # HTML parser. It is here because Supported plugins that an installed gem - # must be able to run need it -- PublishMarkdown, and FeedParser.parse_html - # for SubscriptionLink and SubscriptionTumblr. - spec.add_dependency 'nokogiri', '>= 1.15', '< 2.0' # HTML parsing, in plugins spec.add_dependency 'rexml', '>= 3.2', '< 4.0' # OPML parser spec.add_dependency 'rss', '>= 0.3', '< 1.0' # the pipeline value - spec.add_dependency 'sqlite3', '>= 1.7', '< 3.0' # store plugins spec.add_development_dependency 'rake', '~> 13.0' spec.add_development_dependency 'rspec', '~> 3.13' diff --git a/config/feed2markdown.yml b/config/feed2markdown.yml index e9156b6..07246f4 100644 --- a/config/feed2markdown.yml +++ b/config/feed2markdown.yml @@ -1,14 +1,28 @@ -# Collect, de-duplicate, and leave the result as a Markdown document. +# Collect public information and leave the result as a Markdown document. # # automatic -c ~/.automatic/config/example/feed2markdown.yml # # The document is readable as it stands, searchable with grep, worth keeping in # version control, and ready to hand to a program that takes text as input. -# Nothing here needs an account, a credential or a service. +# Nothing here needs an account, a credential, a service or a gem beyond the +# ones `gem install automatic` brings: the Recipe is the framework's core and +# two plugins that need nothing of their own. # -# Safe to run from cron: StorePermalink records what has already been written, -# so a later run appends only what is new and a run that finds nothing new -# leaves the file untouched. See doc/DEPLOYMENT.md. +# Running it twice appends the same items twice. To collect only what is new, +# put a store plugin in front of the publisher -- it records what has already +# been written, which is also what makes a Recipe safe to run from cron: +# +# - module: StorePermalink +# config: +# db: feed2markdown.db +# +# The store plugins keep their records in SQLite through ActiveRecord, which is +# an optional dependency rather than a framework one: +# +# gem install activerecord sqlite3 +# +# See doc/DEPLOYMENT.md for the optional plugin dependencies, and +# doc/QUICKSTART.md for this Recipe step by step. # # To send the document to standard output instead, drop the PublishMarkdown # config and set the log level to none, so that only the document is written: @@ -21,10 +35,6 @@ plugins: feeds: - https://www.ruby-lang.org/en/feeds/news.rss - - module: StorePermalink - config: - db: feed2markdown.db - - module: PublishMarkdown config: file: ~/.automatic/markdown/feeds.md diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index 4d398a9..f315326 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -32,14 +32,15 @@ nothing to stop. a version between them is supported and is simply not checked on every commit, and a Ruby newer than 4.0 is permitted rather than refused. See [`REQUIREMENTS.md`](REQUIREMENTS.md) section 20. -- A build environment may be needed if a dependency such as `nokogiri` or - `sqlite3` has to build a native extension on your platform. Start with the - normal installation below; install platform-specific build tools only if the - gem installation reports that they are required. - - Optional gems for particular plugins, listed in the table under - "Optional plugin dependencies" below. None is needed to install or to run a - Recipe that does not use the plugin. + "Optional plugin dependencies" below. None is needed to install Automatic + Ruby, to run the Quick Start, or to run a Recipe that does not use the + plugin. +- A build environment may be needed for one of those optional gems — `nokogiri` + and `sqlite3` build a native extension where no binary package matches your + platform. The framework's own dependencies are pure Ruby, so the normal + installation needs no build tools; install them only if installing an + optional gem reports that they are required. ## Install @@ -50,12 +51,29 @@ gem install automatic automatic --version ``` -That installs the framework, its runtime dependencies and the `automatic` -command. +That installs the framework, the `automatic` command and four pure-Ruby +runtime dependencies: `activesupport`, `hashie`, `rexml` and `rss`. That is the +whole of it. No HTML parser, no database, no service client — a gem needed by +one plugin is installed by the operator who uses that plugin, so installing +Automatic Ruby does not install what your Recipes do not use. + +Add one when you use the plugin that needs it: + +```sh +gem install nokogiri # the plugins that read HTML +gem install activerecord sqlite3 # the store plugins +``` + +The table under "Optional plugin dependencies" below says which plugin needs +which, and is the list to check before adding anything. ### From a checkout -For working on the framework, or for running a version that is not released: +For working on the framework, or for running a version that is not released. +There are three ways to set one up, and the first is the one to start with. + +**Minimal — the framework and its test suite.** What you want for running the +checkout, and for developing the framework itself: ```sh git clone https://github.com/id774/automaticruby.git @@ -65,14 +83,44 @@ bundle exec bin/automatic --version bundle exec rake ``` -`bundle install` resolves the runtime and development dependencies in `Gemfile` -and `automatic.gemspec`. It installs what is needed to run the checkout and its -test suite. If the `bundle` command is unavailable, install Bundler first with -`gem install bundler`. +`bundle install` resolves the runtime dependencies of `automatic.gemspec` and +the development ones — `rake`, `rspec` and `simplecov`. It installs **no** +optional plugin gem: the `Gemfile`'s groups for those are optional, and Bundler +does not install an optional group unless it is asked to. If the `bundle` +command is unavailable, install Bundler first with `gem install bundler`. + +**All supported optional plugin dependencies.** For plugin development, or for +running the specs of the plugins that need a gem: + +```sh +bundle config set --local with plugins +bundle install +bundle exec rake +``` + +That adds `activerecord`, `sqlite3`, `nokogiri`, `sanitize`, `nkf` and +`feedbag`, and their specs then run as part of the ordinary suite. The setting +is written to the checkout's own `.bundle/config`, which is not committed; +`bundle config unset --local with` returns the checkout to the minimum, and +`bundle install` afterwards. -The default bundle does not install the optional `plugins` group. See -"Optional plugin dependencies" below and [`PLUGINS.md`](PLUGINS.md) before -installing anything for a particular plugin. +**One dependency at a time.** Start minimal and add only what a plugin you +actually use needs. Each optional gem is in a second, smaller group named after +what it is for, so the group name selects it on its own: + +```sh +bundle config set --local with store # activerecord and sqlite3 +bundle install +``` + +Several at once are space-separated: `bundle config set --local with "store +html"`. The group names are in the table below. + +In a checkout, `gem install ` on its own is **not** enough: `bundle exec` +puts only the bundle on the load path, so a gem the `Gemfile` does not mention +is not visible to it. Use the group, which is why the groups exist. Outside a +checkout — the installed gem, run as `automatic` — there is no bundle and `gem +install ` is exactly right. Everything below that says `automatic` becomes `bundle exec bin/automatic` in a checkout. @@ -116,7 +164,10 @@ worth telling apart: - A message about the feed being unreachable means the network or that particular feed, not the installation. -- A Ruby `LoadError` naming a gem means the installation. +- A `LoadError` naming a gem means a plugin's optional dependency is not + installed. `feed2console.yml` uses none, so at this point it means the + installation itself; a message naming a gem and a plugin means the plugin, + and "Optional plugin dependencies" below says what to install. To check the framework without any network at all, write a Recipe that uses `SubscriptionText`: @@ -176,6 +227,11 @@ plugins: interval: 3 ``` +That Recipe needs three optional gems, because of the plugins it names rather +than because of the framework: `nokogiri` for `FilterImageSource`, and +`activerecord` and `sqlite3` for `StorePermalink`. See "Optional plugin +dependencies" below. + Three things in that Recipe are the operational advice of this document: - **`StorePermalink` before the plugin with the effect.** It records what has @@ -387,34 +443,43 @@ Recipe is stated there in terms you can act on. ## Optional plugin dependencies -These gems are not installed with the framework. Install one only if you use the -plugin, and check its status in [`PLUGINS.md`](PLUGINS.md) section 6 first — -several of these plugins talk to services that no longer exist. - -| Plugin | Needs | Status | -| --- | --- | --- | -| `FilterSanitize` | `sanitize` | Supported | -| `FilterDescriptionLink` | `nkf` | Supported | -| `CustomFeedSVNLog` | `xml-simple`, and the `svn` command | Supported (external) | -| `ProvideFluentd`, `PublishFluentd` | `fluent-logger`, and a Fluentd instance | Supported (external) | -| `PublishMemcached` | `dalli`, and a memcached server | Supported (external) | -| `PublishEject` | the `eject` or `drutil` command | Supported (external) | -| `NotifyIkachan` | an `ikachan` gateway you run | Supported (external) | -| `StoreFile`, S3 path only | the `aws-sdk` v1 interface | Needs rework | -| `PublishAmazonS3` | the `aws-sdk` v1 interface | Needs rework | -| `PublishTwitter`, `SubscriptionTwitterSearch` | — | Unsupported | -| `PublishPocket`, `SubscriptionPocket` | — | Unsupported | -| `PublishHipchat` | — | Unsupported | -| `PublishGoogleCalendar` | — | Unsupported | -| `SubscriptionWeather` | — | Unsupported | - -```sh -gem install sanitize # for FilterSanitize -gem install nkf # for FilterDescriptionLink -gem install fluent-logger # for the Fluentd plugins -gem install dalli # for PublishMemcached -gem install xml-simple # for CustomFeedSVNLog -``` +This table is the list. Which plugin needs which gem, how to install it, and +whether the plugin still works are all here, and nothing else repeats it. + +None of these gems is installed by `gem install automatic` or by a default +`bundle install`. Install one only if you use the plugin, and check the status +column first — several of these plugins talk to services that no longer exist. + +**Installed gem**: `gem install `. **Checkout**: `bundle config set +--local with ` and `bundle install`, because `bundle exec` sees only the +bundle. `plugins` is every group in the first block at once. + +| Plugin | Needs | Installed gem | Checkout group | Status | +| --- | --- | --- | --- | --- | +| `StorePermalink`, `StoreFullText` | `activerecord`, `sqlite3` | `gem install activerecord sqlite3` | `store` | Supported | +| `FilterImageSource`, `FilterDescriptionLink`, `SubscriptionLink`, `SubscriptionTumblr` | `nokogiri` | `gem install nokogiri` | `html` | Supported (`SubscriptionTumblr` external) | +| `PublishMarkdown` | `nokogiri`, for HTML bodies only | `gem install nokogiri` | `html` | Supported; runs without it | +| `FilterSanitize` | `sanitize` | `gem install sanitize` | `sanitize` | Supported | +| `FilterDescriptionLink` | `nkf`, as well as `nokogiri` | `gem install nkf` | `nkf` | Supported | +| `autodiscovery` and `inspect` subcommands | `feedbag` | `gem install feedbag` | `autodiscovery` | Supported | +| `FilterFullFeed` | `nokogiri`, and a siteinfo file | `gem install nokogiri` | `html` | Supported (external) | +| `CustomFeedSVNLog` | `xml-simple`, and the `svn` command | `gem install xml-simple` | `svn_log` | Supported (external) | +| `ProvideFluentd`, `PublishFluentd` | `fluent-logger`, and a Fluentd instance | `gem install fluent-logger` | `fluentd` | Supported (external) | +| `PublishMemcached` | `dalli`, and a memcached server | `gem install dalli` | `memcached` | Supported (external) | +| `PublishEject` | the `eject` or `drutil` command | — | — | Supported (external) | +| `NotifyIkachan` | an `ikachan` gateway you run | — | — | Supported (external) | +| `StoreFile`, S3 path only | the `aws-sdk` v1 interface | — | — | Needs rework | +| `PublishAmazonS3` | the `aws-sdk` v1 interface | — | — | Needs rework | +| `PublishTwitter`, `SubscriptionTwitterSearch` | — | — | — | Unsupported | +| `PublishPocket`, `SubscriptionPocket` | — | — | — | Unsupported | +| `PublishHipchat` | — | — | — | Unsupported | +| `PublishGoogleCalendar` | — | — | — | Unsupported | +| `SubscriptionWeather` | — | — | — | Unsupported | + +The `plugins` group is the first six rows: the optional gems of the plugins +whose specs need nothing but the gem. The three rows below it are in their own +groups only, because each also needs a service or a command, and installing a +gem alone would not make the plugin — or its spec — work. The two AWS rows are listed for completeness rather than as instructions. They call `AWS::S3`, which AWS SDK for Ruby version 1 provided and the current @@ -422,16 +487,19 @@ They call `AWS::S3`, which AWS SDK for Ruby version 1 provided and the current need rework. `StoreFile` makes that requirement lazily, so its ordinary HTTP download path works with no AWS gem installed at all. -In a checkout, install the `Gemfile`'s optional `plugins` group instead — -uncommenting the entry first, where the gem is one of the commented ones: +No optional group is installed by default and none is installed in required CI, +so these plugins are outside what a green build guarantees. Installing a group +brings the specs of its plugins into the ordinary `bundle exec rake` run, which +is how they are verified. -```sh -BUNDLE_WITH=plugins bundle install -``` +Using a plugin without its gem is not a mystery: the plugin says what is +missing, what needs it and how to get it, and the command exits `1`. -That group is not installed by default and is not installed in CI, so these -plugins are outside what the default test suite verifies. Installing it also -brings their specs into the ordinary `bundle exec rake` run. +```text +automatic: The `activerecord` gem is not installed. It is needed by the store +plugins StorePermalink and StoreFullText. Install it with `gem install +activerecord`, ... +``` ## Your own plugins @@ -453,9 +521,11 @@ not touch this directory. **`command not found: automatic`** — the gem's binary directory is not on `PATH`. `gem environment` prints it as EXECUTABLE DIRECTORY. -**`LoadError: cannot load such file -- `** — a plugin's optional dependency -is missing. The message names it; install it, or check the table above in case -the plugin is one that no longer works. +**`The gem is not installed. It is needed by ...`** — a plugin's optional +dependency is missing. The message names the gem, the plugin and the command to +install it; the table above says the same thing, and says whether the plugin is +one that no longer works. A bare `LoadError: cannot load such file -- ` is +the same situation from a plugin that is no longer supported. **`unknown plugin named X`** — the Recipe names a module the loader cannot resolve. Check the spelling against [`PLUGINS.md`](PLUGINS.md) section 6, and diff --git a/doc/PLUGINS.md b/doc/PLUGINS.md index 2c3bd32..3a83a6d 100644 --- a/doc/PLUGINS.md +++ b/doc/PLUGINS.md @@ -378,18 +378,30 @@ secret file. A plugin therefore: ### 3.8 Dependencies -A plugin requires its own libraries at the top of its own file: +A plugin requires its own libraries at the top of its own file. A library that +ships with Ruby is required plainly; a gem the operator has to install is +required through `Automatic.require_optional`, which names the gem, the plugin +and the way to install it if it is absent: ```ruby module Automatic::Plugin class PublishMemcached - require 'dalli' + Automatic.require_optional('dalli', needed_by: 'PublishMemcached') ``` +```text +The `dalli` gem is not installed. It is needed by PublishMemcached. Install it +with `gem install dalli`, or in a source checkout add its group to the bundle; +see the optional plugin dependencies in doc/DEPLOYMENT.md. +``` + +Pass `gem_name:` where the gem's name differs from the path required, as +`xml-simple` does from `xmlsimple`. + That is what keeps a gem needed by one plugin out of everyone else's installation. A gem used by a single plugin is not added to the framework's -runtime dependencies; it goes in the `Gemfile`'s optional `:plugins` group and -the operator who uses the plugin installs it. See [`POLICY.md`](POLICY.md) +runtime dependencies; it goes in an optional group of the `Gemfile` and the +operator who uses the plugin installs it. See [`POLICY.md`](POLICY.md) section 9. Where a plugin has an optional capability that needs a heavier library — S3 @@ -522,13 +534,15 @@ Two rules govern this table, and they are the reason it exists at all: framework was used for, and several remain useful as templates for a replacement. Removal is a separate, deliberate decision. -**Supported is not the same as covered by CI.** A Supported plugin whose gem is -an optional plugin dependency — `FilterSanitize` and `FilterDescriptionLink` — -works, and is simply not part of what a green build guarantees, because the -default bundle does not install that gem. Its entry says so, and installing the -gem runs its spec as part of the ordinary suite. Nothing here is classified by -what CI happens to run; a plugin is not demoted for needing a gem, and is not -promoted by a test that CI never executes. +**Supported is not the same as covered by the required workflow.** A Supported +plugin whose gem is an optional plugin dependency — the store plugins, the ones +that read HTML, `FilterSanitize`, `FilterDescriptionLink` — works, and is +simply not part of what a green required build guarantees, because the default +bundle does not install that gem. Its entry says so, and installing the gem runs +its spec as part of the ordinary suite, which is also what the separate +`plugins` workflow does. Nothing here is classified by what CI happens to run; a +plugin is not demoted for needing a gem, and is not promoted by a test that CI +never executes. **This classification is a snapshot taken in August 2026,** based on the published status of each service and on what each plugin's code actually calls. @@ -586,6 +600,9 @@ incoming pipeline. Set `interval` when fetching several pages from one host. +Reads HTML through `FeedParser.parse_html`, so it needs `nokogiri`: +`gem install nokogiri`, or the `html` group in a checkout. + #### SubscriptionXml — **Supported** `subscription/xml.rb`. `GET`s an XML endpoint, converts the document to a hash, @@ -625,9 +642,10 @@ drops any that leave the blog's own host. `pages` walks `/page/2` and onward. | `retry` | integer | Attempts after the first. Default `0`. | | `interval` | integer | Seconds between requests. Default `0`. | -It reads HTML written for a browser, so it depends on the theme a given blog -uses and on Tumblr's page structure. Verify against the blog you mean to follow -before putting it in `cron`, and set `interval`. +It reads HTML written for a browser, so it needs `nokogiri` — `gem install +nokogiri`, or the `html` group in a checkout — and it depends on the theme a +given blog uses and on Tumblr's page structure. Verify against the blog you mean +to follow before putting it in `cron`, and set `interval`. #### SubscriptionTwitter — **Unsupported** @@ -753,6 +771,8 @@ settings. `` values in the description, or, if there are none, the images on the page the link points at. Fetching pages means network access. No settings. +Needs `nokogiri`: `gem install nokogiri`, or the `html` group in a checkout. + #### FilterAbsoluteURI — **Supported** `filter/absolute_uri.rb`. Rewrites relative links to absolute ones. @@ -794,11 +814,12 @@ the body. `get_title` makes one request per item; use `FilterOne` or a store plugin before it on a large feed. -Needs the `nkf` gem, which the plugin uses to normalize a fetched page's -encoding. `nkf` left the standard library after Ruby 3.3 and is an optional -plugin dependency rather than a framework one, so it is not installed with the -framework, and this plugin's spec is outside the default suite and outside CI. -See [`DEPLOYMENT.md`](DEPLOYMENT.md). +Needs `nkf`, which the plugin uses to normalize a fetched page's encoding, and +`nokogiri`, which it reads the fetched page with. `nkf` left the standard +library after Ruby 3.3; both are optional plugin dependencies rather than +framework ones, so neither is installed with the framework, and this plugin's +spec is outside the default suite and outside the required workflow. See +[`DEPLOYMENT.md`](DEPLOYMENT.md). #### FilterFullFeed — **Supported (external)** @@ -810,6 +831,8 @@ page. | --- | --- | --- | | `siteinfo` | string | File name under the assets directory. Required. | +Needs `nokogiri`: `gem install nokogiri`, or the `html` group in a checkout. + The shipped `assets/siteinfo/items_all.json` is a snapshot of the LDRFullFeed database taken from `wedata.net`, which no longer operates, so the file cannot be refreshed from its origin and its newest entries are from 2013. The plugin @@ -835,6 +858,12 @@ plugin runs without error and simply matches nothing. Persist, and drop what has already been seen. A store plugin is what makes a Recipe safe to run repeatedly. +`StorePermalink` and `StoreFullText` keep their records in SQLite through +ActiveRecord. Both gems are these plugins' own optional dependencies rather than +the framework's: `gem install activerecord sqlite3`, or the `store` group in a +checkout. A Recipe that stores nothing needs neither. +See [`DEPLOYMENT.md`](DEPLOYMENT.md). + #### StorePermalink — **Supported** `store/permalink.rb`. Records each item's link in SQLite and passes on only the @@ -1017,15 +1046,18 @@ This is deliberately **not** an HTML-to-Markdown translation. Rendering arbitrary markup back into equivalent Markdown — tables, nested lists, inline links, images — is a large job with a large library behind it, and a library that size does not become a dependency for one plugin -([`POLICY.md`](POLICY.md) section 9.1). Reducing markup to text needs nothing -beyond `nokogiri`, which `gem install automatic` installs: it is a runtime -dependency of this gem precisely so that the Supported plugins an installed gem -must be able to run — this one, and `FeedParser.parse_html` for -`SubscriptionLink` and `SubscriptionTumblr` — work with nothing else added. -Requiring `automatic` itself loads no HTML parser. The result is defined by its -two ends — the text survives, the markup does not — which is what both a reader -and a program reading the file want from it. A link inside a body becomes its -own text; the item's own link is in the metadata list, where nothing loses it. +([`POLICY.md`](POLICY.md) section 9.1). The result is defined by its two ends — +the text survives, the markup does not — which is what both a reader and a +program reading the file want from it. A link inside a body becomes its own +text; the item's own link is in the metadata list, where nothing loses it. + +**This plugin needs no gem of its own.** It uses `nokogiri` to reduce a body +where `nokogiri` is installed, and reduces it with its own substitution where it +is not, so that the Quick Start runs on a plain `gem install automatic` and no +Recipe pays for an HTML parser it did not ask for. The two produce the same +document for the bodies a feed carries; a parser is simply better at markup that +is badly malformed, which is the reason to install `nokogiri` if you publish +from feeds that produce it. The specs hold both to the same output. Where a different treatment is wanted, the pipeline already has the means: `FilterSanitize` before this plugin decides what markup survives into the diff --git a/doc/POLICY.md b/doc/POLICY.md index b2dab9c..f6c6e16 100644 --- a/doc/POLICY.md +++ b/doc/POLICY.md @@ -371,6 +371,10 @@ repository level only; see section 10. plugin loads without it. The S3 branch of `StoreFile` is the example. - The CLI requires a subcommand's libraries inside that subcommand, so a command that does not use `feedbag` or the OPML parser does not load them. +- **An optional gem is required through `Automatic.require_optional`**, which + names the gem, what needed it and how to install it when it is absent. A bare + `require` of an optional gem answers a solvable problem with + `cannot load such file`, which is not the framework's best answer. - **`Bundler.require` is not how the framework loads its dependencies.** An installed library must not impose a bundle on the program requiring it. The Bundler setup in `environment.rb` is a convenience for a source checkout and @@ -442,15 +446,19 @@ Plugins outlive the services they talk to. The policy for what happens then: them a gate. A new example that reaches a host is tagged; one that does not is never given the tag to make a failure go away. - **The default suite does not depend on an optional plugin gem.** A gem the - `Gemfile` declares in its optional `:plugins` group is not installed by - `bundle install`, so the plugins that need it are not verified by the default - suite or by CI. That is a decision, taken here, and not something a failure + `Gemfile` declares in an optional group is not installed by `bundle install`, + so the plugins that need it are not verified by the default suite or by the + required workflow. That is a decision, taken here, and not something a failure discovers: the gems it applies to are the declared list `AutomaticSpec::OPTIONAL_PLUGIN_GEMS`, a spec whose plugin needs one guards its file with `AutomaticSpec.optional_dependency?`, and naming a gem that is - not on the list raises rather than skipping. Installing the group with - `BUNDLE_WITH=plugins bundle install` runs those specs as part of the ordinary - suite. + not on the list raises rather than skipping. Selecting the group with + `bundle config set --local with plugins` and installing runs those specs as + part of the ordinary suite. +- **A guard is a skip with a reason, not a `pending`.** A spec outside the + default suite prints why it is outside it and does not run; the default + suite's output is a list of what was verified rather than a list of what was + not. - A spec whose plugin's gem is not installed is skipped by `AutomaticSpec.plugin_available?`, which names the missing gem. That is the intended behaviour and is not worked around by faking the gem. @@ -558,28 +566,46 @@ behaviour breaks nothing. ### 9.1 The split -Dependencies fall into three groups, and which group a gem is in is a decision, -not an accident: +Install the core by default; install a plugin's dependencies only when they are +needed. Dependencies fall into three groups, and which group a gem is in is a +decision, not an accident: **Runtime dependencies** — declared in `automatic.gemspec`, installed by `gem -install automatic`. A gem is here only if the framework itself uses it, or if a -plugin that the majority of Recipes use — or that the documented primary -workflow runs — needs it. A gem that reached this list because of a plugin that -no longer uses it, or because a Ruby release moved a library out of the standard -library, is moved back out. +install automatic`. A gem is here only if a file in `lib/` requires it: what +`require 'automatic'`, loading a Recipe, loading a plugin, running a pipeline +and the command line itself need, and nothing more. A gem that reached this list +because of a plugin, because a plugin no longer uses it, or because a Ruby +release moved a library out of the standard library, is moved back out. **Optional dependencies** — used by one plugin or a few, required inside the -plugin's own file, declared in the `Gemfile`'s optional `:plugins` group, and -**not declared as runtime dependencies**. An operator who uses that plugin -installs the gem. This is Invariant 4, and it is why installing this gem does -not install an AWS SDK. - +plugin's own file, declared in an optional group of the `Gemfile`, and **not +declared as runtime dependencies**. An operator who uses that plugin installs +the gem. This is Invariant 4, and it is why installing this gem does not install +an AWS SDK. + +The permanent rules of the split: + +- **A core dependency is one the framework itself has.** A plugin's dependency + is never a framework runtime dependency, however useful, popular or + Supported that plugin is. How many Recipes happen to use it is not the test; + what `lib/` requires is. +- **The operator who uses the plugin installs its gem.** Not having it must + never stop the framework from starting, and a Recipe that does not name the + plugin must run without it. +- **A new plugin does not increase the core dependencies.** Adding one to the + runtime list needs an architectural justification — the framework itself + came to need the gem — recorded with the change; wanting the plugin to work + out of the box is not one. +- **The default tests and required CI do not depend on an optional + integration.** Being in this group has that intended consequence: those + plugins are not part of what a green build guarantees. See section 5. - **An unsupported or optional integration does not decide a framework-wide - dependency.** Where one plugin needs a gem, that gem is the plugin's, however - useful the plugin is. -- Being in this group has a consequence that is intended: the default suite and - CI do not install it, so those plugins are not part of what a green build - guarantees. See section 5. + dependency.** Where one plugin needs a gem, that gem is the plugin's. + +A missing optional gem is reported, not merely raised: the plugin requires it +through `Automatic.require_optional`, which names the gem, what needed it and +how to install it. That helper is the whole of the mechanism, and no dependency +manager, plugin manifest or resolver is introduced beyond it. **Development dependencies** — the test and build tooling. @@ -753,7 +779,10 @@ the version history records what it amounts to. ## 11. Continuous integration -- CI runs on GitHub Actions, from `.github/workflows/ci.yml`. +- CI runs on GitHub Actions. `.github/workflows/ci.yml` is the required check; + `.github/workflows/plugins.yml` is a separate, non-required workflow that + installs the optional `plugins` group and runs the same suite, so that the + documented all-plugins setup is checked as well as described. - **CI validates representative supported Ruby versions rather than every intermediate release.** The matrix runs the ends of the supported range and the release in the middle. The matrix and `required_ruby_version` are @@ -767,9 +796,14 @@ the version history records what it amounts to. integration is not added to it. - **CI holds no secret and reaches no external service.** No credential is configured, and no integration test against a third-party API is run there. -- **CI installs no optional plugin gem**, so no plugin's own dependency is a - condition of a green build. Where an optional integration is worth testing at - all, it is tested separately from the required workflow; section 5 says how. +- **The required workflow installs no optional plugin gem**, so no plugin's own + dependency is a condition of the check that gates a change. The minimal + configuration is what it runs, which is what keeps the split of section 9.1 + honest over time. Where an optional integration is worth testing at all, it is + tested separately from the required workflow; section 5 says how. +- **A non-required workflow is held to the same standard as the required one.** + It is separate so that an optional dependency cannot gate a change, not so + that it may fail quietly. - **A failure is fixed, not silenced.** `|| true`, `continue-on-error` and a step that hides its exit status are not how a build is made green. Narrowing what is guaranteed is a legitimate answer; pretending to guarantee it is not. diff --git a/doc/QUICKSTART.md b/doc/QUICKSTART.md index 92df1cc..6b842bf 100644 --- a/doc/QUICKSTART.md +++ b/doc/QUICKSTART.md @@ -2,7 +2,8 @@ This guide takes public information through one short Automatic Ruby pipeline and leaves it as Markdown. It needs no account, credential, paid service or -database server, and no gem beyond the ones `gem install automatic` brings. +database server, and no gem beyond the four `gem install automatic` brings — +no HTML parser, no database, nothing to build. ## 1. Install @@ -35,10 +36,6 @@ plugins: feeds: - https://www.ruby-lang.org/en/feeds/news.rss - - module: StorePermalink - config: - db: feed2markdown.db - - module: PublishMarkdown config: file: ~/.automatic/markdown/feeds.md @@ -51,9 +48,9 @@ Run the scaffolded copy: automatic -c ~/.automatic/config/example/feed2markdown.yml ``` -`SubscriptionFeed` acquires the public Ruby news feed. `StorePermalink` keeps a -local SQLite record and passes on only unseen items. `PublishMarkdown` appends -those items to a plain-text document. +`SubscriptionFeed` acquires the public Ruby news feed. `PublishMarkdown` +appends those items to a plain-text document, reducing the HTML in each item's +body to text as it goes. ## 4. Read the result @@ -72,10 +69,46 @@ Each item is a level-2 heading followed by available metadata and a text body: Item body. ``` -Run the Recipe again. Items already recorded by `StorePermalink` are not -appended again. +Run the Recipe again, and the same items are appended a second time: nothing in +this Recipe remembers what it has already published. The next step is what +fixes that. -## 5. Run it from cron +## 5. Collect only what is new + +A store plugin records what has been published and passes on only what has not. +`StorePermalink` keeps that record in SQLite through ActiveRecord, and those two +gems are the store plugins' own dependencies rather than the framework's, so +they are installed when they are wanted: + +```sh +gem install activerecord sqlite3 +``` + +Then put the plugin between the two the Recipe already has: + +```yaml +plugins: + - module: SubscriptionFeed + config: + feeds: + - https://www.ruby-lang.org/en/feeds/news.rss + + - module: StorePermalink + config: + db: feed2markdown.db + + - module: PublishMarkdown + config: + file: ~/.automatic/markdown/feeds.md + mode: append +``` + +Run it twice. The second run appends nothing, which is what makes the Recipe +safe to run from `cron` — and, in general, what to do before any plugin with an +effect. Other plugins have optional dependencies of their own, all listed in +[`DEPLOYMENT.md`](DEPLOYMENT.md). + +## 6. Run it from cron Create the log directory once, then use the absolute path reported by `command -v automatic`: @@ -103,6 +136,11 @@ bundle exec bin/automatic scaffold bundle exec bin/automatic -c ~/.automatic/config/example/feed2markdown.yml ``` +A checkout resolves gems through Bundler rather than through RubyGems, so step 5 +is done differently there: `bundle config set --local with store` and +`bundle install`, instead of `gem install activerecord sqlite3`. See +[`DEPLOYMENT.md`](DEPLOYMENT.md). + The Recipe is ordinary YAML. Change the feed URL, insert a supported Filter, or change the Markdown path without changing the framework. To write a small plugin, continue with [`PLUGIN_DEVELOPMENT.md`](PLUGIN_DEVELOPMENT.md). diff --git a/doc/RELEASING.md b/doc/RELEASING.md index dba40fc..a9ecfe9 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -210,7 +210,9 @@ Confirm that: - the required Ruby version is `>= 3.3.0`; - licenses contain both `GPL-3.0-only` and `LGPL-3.0-only`; - the authors are correct; -- runtime and development dependencies match `automatic.gemspec`; +- runtime and development dependencies match `automatic.gemspec`, and the + runtime ones are the framework's own — no gem that belongs to a plugin has + found its way in ([`POLICY.md`](POLICY.md) section 9.1); - `rubygems_mfa_required` is `true`. ## 8. Test an isolated local installation diff --git a/doc/REQUIREMENTS.md b/doc/REQUIREMENTS.md index ef9006a..2f9542c 100644 --- a/doc/REQUIREMENTS.md +++ b/doc/REQUIREMENTS.md @@ -418,7 +418,9 @@ Requirements: ActiveRecord is used as a library and this is not a Rails application; nothing shall introduce one. Equally, ActiveRecord shall not be replaced by a hand-written database layer merely because it is a large dependency — it is what the existing databases were written by, and operators -have those files. +have those files. It is the store plugins' dependency and not the framework's: +`activerecord` and `sqlite3` shall be installed by the operator who uses those +plugins, and a Recipe that stores nothing shall run without either. **The filesystem.** `StoreFile` downloads what the pipeline points at and rewrites the item's link to a `file://` URI, which is how a later publishing diff --git a/doc/VERSIONS b/doc/VERSIONS index 3a091b7..fa34225 100644 --- a/doc/VERSIONS +++ b/doc/VERSIONS @@ -7,7 +7,7 @@ v26.08 (Release Date: TBD) - Harden Recipe loading with safe YAML parsing, structural validation and framework-specific errors. - Restructure the CLI with help and version options, predictable error reporting and documented exit statuses. - Verify TLS certificates when publishing to Instapaper instead of accepting an unverified connection. -- Modernize gem packaging and dependency policy, isolating optional plugin dependencies outside the framework and the default test path, and excluding development and generated files. +- Modernize gem packaging and dependency policy, separating core requirements from optional plugin dependencies so that an installation can be minimal, complete or extended one plugin at a time, and excluding development and generated files. - Classify every shipped plugin by its current support status rather than simulating obsolete services in tests. - Rebuild the test and CI strategy for current RSpec and Ruby, with deterministic isolation from user data and external services. - Add Markdown as the primary service-independent publication format, with a documented and tested first-run workflow. diff --git a/lib/automatic.rb b/lib/automatic.rb index facb2de..7e5f4d0 100755 --- a/lib/automatic.rb +++ b/lib/automatic.rb @@ -40,6 +40,27 @@ class InvalidRecipeError < Error; end class << self attr_accessor :root_dir + # Require a gem that only one plugin, or one optional path, needs, and turn + # its absence into a message naming the gem, what wanted it and how to get + # it. Nothing here resolves, installs or tracks a dependency: the `require` + # is the plugin's own, made where the plugin makes it, and this only + # replaces `cannot load such file -- nkf` with a sentence an operator can + # act on. See doc/POLICY.md section 9.1 and doc/PLUGINS.md section 3.8. + # + # Automatic.require_optional('sanitize', needed_by: 'FilterSanitize') + # + # `gem_name` is given where it differs from the path required, as + # `xmlsimple` does from the `xml-simple` gem. + def require_optional(feature, needed_by:, gem_name: feature) + require feature + rescue LoadError => e + raise LoadError, + "The `#{gem_name}` gem is not installed. It is needed by #{needed_by}. " \ + "Install it with `gem install #{gem_name}`, or in a source checkout add " \ + 'its group to the bundle; see the optional plugin dependencies in ' \ + "doc/DEPLOYMENT.md. (#{e.message})" + end + # Run one Recipe. root_dir is the installation root; user_dir is honoured # only under AUTOMATIC_RUBY_ENV=test, which is how the specs point the # plugin loader at a fixture directory. diff --git a/lib/automatic/cli.rb b/lib/automatic/cli.rb index 8015da7..829987a 100644 --- a/lib/automatic/cli.rb +++ b/lib/automatic/cli.rb @@ -103,6 +103,11 @@ def usage(parser) EXIT_FAILURE end + # A LoadError is one of the failures reported as a message here: a Recipe + # naming a plugin whose optional gem is not installed is an operator's + # mistake with an answer, and the answer is in the message rather than in a + # backtrace. What the plugin itself raises is left alone; see + # doc/PLUGINS.md section 2.7. def run_recipe(path) unless File.exist?(resolve_recipe(path)) @stderr.puts "automatic: no such recipe: #{path}" @@ -111,7 +116,7 @@ def run_recipe(path) Automatic.run(recipe: Automatic::Recipe.new(path), root_dir: @root_dir) EXIT_SUCCESS - rescue Automatic::Error, Psych::Exception, SystemCallError, IOError => e + rescue Automatic::Error, Psych::Exception, SystemCallError, IOError, LoadError => e @stderr.puts "automatic: #{e.message}" EXIT_FAILURE end @@ -134,7 +139,7 @@ def run_subcommand(argv) handler.call(argv) EXIT_SUCCESS - rescue Automatic::Error, SystemCallError, IOError => e + rescue Automatic::Error, SystemCallError, IOError, LoadError => e @stderr.puts "automatic: #{e.message}" EXIT_FAILURE end @@ -146,6 +151,11 @@ def missing_argument(name) # Each subcommand requires what it needs, so that `automatic --version` # loads neither a feed parser nor an OPML parser. + # + # `feedbag` is an optional dependency: two subcommands out of seven use it, + # and neither running a Recipe nor any other subcommand does. It is + # required through Automatic.require_optional, so that not having it is a + # sentence rather than a backtrace. See doc/POLICY.md section 9.1. def subcommands { 'scaffold' => method(:scaffold), @@ -189,7 +199,7 @@ def unscaffold(_argv) end def autodiscovery(argv) - require 'feedbag' + Automatic.require_optional('feedbag', needed_by: 'the autodiscovery subcommand') require 'pp' url = argv.shift || missing_argument('autodiscovery') @stdout.puts Feedbag.find(url).pretty_inspect @@ -204,7 +214,7 @@ def feedparser(argv) def inspect_url(argv) require 'automatic/feed_parser' - require 'feedbag' + Automatic.require_optional('feedbag', needed_by: 'the inspect subcommand') require 'pp' url = argv.shift || missing_argument('inspect') feeds = Feedbag.find(url) diff --git a/lib/automatic/feed_parser.rb b/lib/automatic/feed_parser.rb index 921a516..7d10c2e 100644 --- a/lib/automatic/feed_parser.rb +++ b/lib/automatic/feed_parser.rb @@ -33,10 +33,17 @@ def self.get_url(url) # a page that publishes no feed enters the pipeline. # # nokogiri is required here rather than at the top of the file: it is the - # only thing in the framework that wants an HTML parser, and requiring - # `automatic` should not load one. See doc/POLICY.md section 2.5. + # only thing in the framework that wants an HTML parser, it is an optional + # dependency rather than a runtime one, and requiring `automatic` must + # neither load one nor need one installed. Only the plugins that call this + # method -- SubscriptionLink and SubscriptionTumblr -- do. + # See doc/POLICY.md sections 2.5 and 9.1. def self.parse_html(html) - require 'nokogiri' + Automatic.require_optional( + 'nokogiri', + needed_by: 'Automatic::FeedParser.parse_html, used by SubscriptionLink ' \ + 'and SubscriptionTumblr' + ) RSS::Maker.make('2.0') do |maker| maker.xml_stylesheets.new_xml_stylesheet diff --git a/plugins/custom_feed/svn_log.rb b/plugins/custom_feed/svn_log.rb index 3413ede..37ab2ff 100644 --- a/plugins/custom_feed/svn_log.rb +++ b/plugins/custom_feed/svn_log.rb @@ -5,14 +5,16 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 29, 2012 -# Updated:: Mar 3, 2012 +# Updated:: Aug 14, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require 'rss/maker' module Automatic::Plugin class CustomFeedSVNLog - require 'xmlsimple' + Automatic.require_optional('xmlsimple', + gem_name: 'xml-simple', + needed_by: 'CustomFeedSVNLog') def initialize(config, pipeline=[]) @config = config diff --git a/plugins/filter/description_link.rb b/plugins/filter/description_link.rb index 894a4fa..2ee7b5d 100644 --- a/plugins/filter/description_link.rb +++ b/plugins/filter/description_link.rb @@ -11,8 +11,8 @@ module Automatic::Plugin class FilterDescriptionLink require 'erb' - require 'nkf' - require 'nokogiri' + Automatic.require_optional('nkf', needed_by: 'FilterDescriptionLink') + Automatic.require_optional('nokogiri', needed_by: 'FilterDescriptionLink') require 'open-uri' require 'uri' diff --git a/plugins/filter/full_feed.rb b/plugins/filter/full_feed.rb index abc7207..a3efe25 100644 --- a/plugins/filter/full_feed.rb +++ b/plugins/filter/full_feed.rb @@ -12,7 +12,7 @@ module Automatic::Plugin class FilterFullFeed require 'json' - require 'nokogiri' + Automatic.require_optional('nokogiri', needed_by: 'FilterFullFeed') require 'open-uri' require 'uri' diff --git a/plugins/filter/image_source.rb b/plugins/filter/image_source.rb index ff017cf..e766c27 100644 --- a/plugins/filter/image_source.rb +++ b/plugins/filter/image_source.rb @@ -11,7 +11,7 @@ module Automatic::Plugin class FilterImageSource require 'net/http' - require 'nokogiri' + Automatic.require_optional('nokogiri', needed_by: 'FilterImageSource') require 'open-uri' require 'uri' diff --git a/plugins/filter/sanitize.rb b/plugins/filter/sanitize.rb index 5318f41..4eec858 100644 --- a/plugins/filter/sanitize.rb +++ b/plugins/filter/sanitize.rb @@ -10,7 +10,7 @@ module Automatic::Plugin class FilterSanitize - require 'sanitize' + Automatic.require_optional('sanitize', needed_by: 'FilterSanitize') def initialize(config, pipeline=[]) @config = config diff --git a/plugins/provide/fluentd.rb b/plugins/provide/fluentd.rb index 91eda5c..fc2ff67 100644 --- a/plugins/provide/fluentd.rb +++ b/plugins/provide/fluentd.rb @@ -5,12 +5,12 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jul 12, 2013 -# Updated:: May 16, 2014 +# Updated:: Aug 14, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class ProvideFluentd - require 'fluent-logger' + Automatic.require_optional('fluent-logger', needed_by: 'ProvideFluentd') def initialize(config, pipeline=[]) @config = config diff --git a/plugins/publish/fluentd.rb b/plugins/publish/fluentd.rb index 3d9263d..2c23e5f 100644 --- a/plugins/publish/fluentd.rb +++ b/plugins/publish/fluentd.rb @@ -5,12 +5,12 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 21, 2013 -# Updated:: Feb 25, 2014 +# Updated:: Aug 14, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishFluentd - require 'fluent-logger' + Automatic.require_optional('fluent-logger', needed_by: 'PublishFluentd') def initialize(config, pipeline=[]) @config = config diff --git a/plugins/publish/markdown.rb b/plugins/publish/markdown.rb index f20f304..22562c8 100644 --- a/plugins/publish/markdown.rb +++ b/plugins/publish/markdown.rb @@ -12,7 +12,6 @@ module Automatic::Plugin class PublishMarkdown require 'fileutils' - require 'nokogiri' # Written under an item's heading, in this order. A field the item does not # carry produces no bullet at all, so an item is not padded out with empty @@ -40,6 +39,28 @@ class PublishMarkdown # plain-text description is not parsed away. MARKUP = %r{<[a-zA-Z/!]|&[a-zA-Z#][0-9a-zA-Z]*;} + # What the substitute reducer below matches, in the order it applies them. + COMMENT_ELEMENT = //m + DISCARDED_ELEMENT = %r{<(script|style)\b[^>]*>.*?}mi + BREAK_ELEMENT = /]*>/i + BLOCK_ELEMENT = %r{]*>}i + TAG = /<[^>]*>/m + ENTITY = /&(\#\d+|\#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/ + + # The character references a feed body actually tends to carry: the five of + # XML, and the punctuation and symbols that follow prose out of a web page. + # A reference in neither this table nor the numeric form is left as it is + # written, which a reader can still make sense of; the full HTML set is two + # thousand entries and a parser's business. + ENTITIES = { + 'amp' => '&', 'lt' => '<', 'gt' => '>', 'quot' => '"', 'apos' => "'", + 'nbsp' => "\u00A0", 'copy' => '©', 'reg' => '®', 'trade' => '™', + 'hellip' => '…', 'mdash' => '—', 'ndash' => '–', 'middot' => '·', + 'bull' => '•', 'deg' => '°', 'laquo' => '«', 'raquo' => '»', + 'lsquo' => '‘', 'rsquo' => '’', 'ldquo' => '“', 'rdquo' => '”', + 'yen' => '¥', 'pound' => '£', 'euro' => '€' + }.freeze + # The item's own date, in the zone the item carries. Nothing is converted # to local time: the same pipeline then produces the same document # wherever it runs. @@ -146,9 +167,66 @@ def url?(text) end # Reduce markup to text: script and style go with their contents, a break - # ends a line, a block element ends a paragraph, entities are decoded by - # the parser, and the tags themselves are dropped. + # ends a line, a block element ends a paragraph, entities are decoded, and + # the tags themselves are dropped. + # + # A parser does it where one is installed, and the substitution below does + # it where none is. That is what keeps this plugin -- the one the Quick + # Start publishes with -- runnable on a plain `gem install automatic`: + # nokogiri is an optional dependency, and reducing a feed body to text is + # not a good enough reason to make everyone install a native extension. + # The two agree on what a body is reduced to; a parser is simply better at + # markup that is malformed. See doc/PLUGINS.md section 6.7. def html_to_text(html) + html_parser? ? parsed_text(html) : substituted_text(html) + end + + # Memoized, and false rather than an exception when the gem is absent: this + # question is asked once per body. + def html_parser? + return @html_parser unless @html_parser.nil? + + @html_parser = + begin + require 'nokogiri' + true + rescue LoadError + false + end + end + + def substituted_text(html) + text = html.gsub(COMMENT_ELEMENT, '') + text = text.gsub(DISCARDED_ELEMENT, '') + text = text.gsub(BREAK_ELEMENT, "\n") + text = text.gsub(BLOCK_ELEMENT, "\n\n") + unescape(text.gsub(TAG, '')) + end + + # One pass, so that "&lt;" decodes to "<" and not to "<", which is + # what a parser does with it too. + def unescape(text) + text.gsub(ENTITY) {|reference| + name = Regexp.last_match(1) + name.start_with?('#') ? character(name) || reference + : ENTITIES.fetch(name, reference) + } + end + + # A numeric character reference, or nil where it names no character: a + # surrogate, a value past the last code point, or zero. + def character(name) + code = name.start_with?('#x', '#X') ? name[2..].to_i(16) : name[1..].to_i + return nil unless code.positive? + + begin + code.chr(Encoding::UTF_8) + rescue RangeError + nil + end + end + + def parsed_text(html) fragment = Nokogiri::HTML.fragment(html) fragment.css('script, style').each {|node| node.remove } fragment.css('br').each {|node| node.replace(text_node(node, "\n")) } diff --git a/plugins/publish/memcached.rb b/plugins/publish/memcached.rb index 6805e81..ec005db 100644 --- a/plugins/publish/memcached.rb +++ b/plugins/publish/memcached.rb @@ -5,12 +5,12 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 25, 2013 -# Updated:: Jun 25, 2013 +# Updated:: Aug 14, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishMemcached - require 'dalli' + Automatic.require_optional('dalli', needed_by: 'PublishMemcached') def initialize(config, pipeline=[]) @config = config diff --git a/plugins/store/database.rb b/plugins/store/database.rb index fde7920..ea4578a 100644 --- a/plugins/store/database.rb +++ b/plugins/store/database.rb @@ -5,10 +5,18 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 27, 2012 -# Updated:: Oct 09, 2014 +# Updated:: Aug 14, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +# +# The SQLite storage the store plugins share. ActiveRecord and sqlite3 are the +# store plugins' own dependencies, not the framework's: a Recipe that stores +# nothing runs without them. See doc/POLICY.md section 9.1. -require 'active_record' +Automatic.require_optional('active_record', + gem_name: 'activerecord', + needed_by: 'the store plugins StorePermalink and StoreFullText') +Automatic.require_optional('sqlite3', + needed_by: 'the store plugins StorePermalink and StoreFullText') module Automatic::Plugin module Database diff --git a/plugins/subscription/twitter.rb b/plugins/subscription/twitter.rb index d728dda..138f334 100644 --- a/plugins/subscription/twitter.rb +++ b/plugins/subscription/twitter.rb @@ -11,7 +11,7 @@ module Automatic::Plugin class SubscriptionTwitter require 'open-uri' - require 'nokogiri' + Automatic.require_optional('nokogiri', needed_by: 'SubscriptionTwitter') require 'rss' def initialize(config, pipeline=[]) diff --git a/spec/config/feed2markdown_spec.rb b/spec/config/feed2markdown_spec.rb index 965b375..178b4b8 100644 --- a/spec/config/feed2markdown_spec.rb +++ b/spec/config/feed2markdown_spec.rb @@ -10,7 +10,16 @@ it 'uses the short supported pipeline documented by the Quick Start' do modules = recipe.fetch('plugins').map { |plugin| plugin.fetch('module') } - expect(modules).to eq %w[SubscriptionFeed StorePermalink PublishMarkdown] + expect(modules).to eq %w[SubscriptionFeed PublishMarkdown] + end + + # The Quick Start is what a plain `gem install automatic` can run, so the + # Recipe it ships names no plugin that needs an optional gem. The store + # plugins, which do, are the documented next step rather than the first one. + # See doc/POLICY.md section 9.1. + it 'names no plugin that needs an optional dependency' do + modules = recipe.fetch('plugins').map { |plugin| plugin.fetch('module') } + expect(modules).not_to include('StorePermalink', 'StoreFullText') end it 'uses a public HTTPS source and needs no credential' do diff --git a/spec/lib/automatic/cli_spec.rb b/spec/lib/automatic/cli_spec.rb index fe838db..4843a57 100644 --- a/spec/lib/automatic/cli_spec.rb +++ b/spec/lib/automatic/cli_spec.rb @@ -11,6 +11,7 @@ require File.expand_path(File.join(File.dirname(__FILE__), '../../spec_helper')) require 'automatic/cli' +require 'fileutils' require 'stringio' require 'tmpdir' @@ -125,6 +126,35 @@ def run(*argv, root_dir: APP_ROOT) end end + # A plugin's optional gem is installed by the operator who uses the plugin + # (doc/POLICY.md section 9.1), so not having one is an ordinary situation + # with an answer. It is reported as a message naming the gem and the + # plugin, not as a backtrace. + it "reports a plugin's missing optional gem as a message" do + # The user plugin directory of the redirected HOME, named here rather + # than asked of Automatic, so that this writes into the spec's temporary + # home whatever another example has left the user directory set to. + plugins = File.join(File.expand_path("~"), ".automatic", "plugins", "filter") + FileUtils.mkdir_p(plugins) + File.write(File.join(plugins, "needs_gem.rb"), <<~RUBY) + module Automatic::Plugin + class FilterNeedsGem + Automatic.require_optional('automatic_no_such_gem', + needed_by: 'FilterNeedsGem') + end + end + RUBY + + Dir.mktmpdir do |dir| + path = File.join(dir, "recipe.yml") + File.write(path, "plugins:\n - module: FilterNeedsGem\n") + expect(run("-c", path)).to eq Automatic::CLI::EXIT_FAILURE + expect(err.string).to match(/`automatic_no_such_gem` gem is not installed/) + expect(err.string).to match(/FilterNeedsGem/) + expect(err.string).to match(/gem install automatic_no_such_gem/) + end + end + it "fails on malformed YAML without raising" do Dir.mktmpdir do |dir| path = File.join(dir, "recipe.yml") diff --git a/spec/lib/automatic_spec.rb b/spec/lib/automatic_spec.rb index 367aa97..458eeff 100644 --- a/spec/lib/automatic_spec.rb +++ b/spec/lib/automatic_spec.rb @@ -106,4 +106,36 @@ end end + # How a plugin requires a gem that only it needs. The framework's own + # dependencies are not required this way; these are the optional ones, which + # the operator who uses the plugin installs (doc/POLICY.md section 9.1). + describe "#require_optional" do + it "requires the library, as require does" do + expect { + Automatic.require_optional("tmpdir", needed_by: "a spec") + }.not_to raise_error + expect(defined?(Dir.mktmpdir)).to eq "method" + end + + it "names the gem, what needs it and how to install it when it is absent" do + expect { + Automatic.require_optional("automatic_no_such_gem", needed_by: "FilterExample") + }.to raise_error(LoadError, /`automatic_no_such_gem` gem is not installed/) + end + + it "reports the gem's name where it differs from the path required" do + expect { + Automatic.require_optional("automatic_no_such_gem", + gem_name: "automatic-no-such-gem", + needed_by: "CustomFeedExample") + }.to raise_error(LoadError, /gem install automatic-no-such-gem/) + end + + it "names what needed it" do + expect { + Automatic.require_optional("automatic_no_such_gem", needed_by: "FilterExample") + }.to raise_error(LoadError, /needed by FilterExample/) + end + end + end diff --git a/spec/plugins/filter/description_link_spec.rb b/spec/plugins/filter/description_link_spec.rb index 7fc2f2f..06cac9a 100644 --- a/spec/plugins/filter/description_link_spec.rb +++ b/spec/plugins/filter/description_link_spec.rb @@ -10,10 +10,11 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# FilterDescriptionLink needs the nkf gem, which the Gemfile declares in its -# optional :plugins group. The default suite and CI do not install it, so this -# spec runs only where the operator has. See doc/POLICY.md section 5. -if AutomaticSpec.optional_dependency?('nkf') +# FilterDescriptionLink needs the nkf and nokogiri gems, which the Gemfile +# declares in its optional :plugins group. The default suite and CI do not +# install them, so this spec runs only where the operator has. See +# doc/POLICY.md section 5. +if AutomaticSpec.optional_dependency?('nkf') && AutomaticSpec.optional_dependency?('nokogiri') require 'filter/description_link' describe Automatic::Plugin::FilterDescriptionLink do diff --git a/spec/plugins/filter/full_feed_spec.rb b/spec/plugins/filter/full_feed_spec.rb index cb305af..ce36e15 100644 --- a/spec/plugins/filter/full_feed_spec.rb +++ b/spec/plugins/filter/full_feed_spec.rb @@ -10,6 +10,11 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') +# FilterFullFeed reads HTML with nokogiri, which the Gemfile declares in its +# optional :plugins group. The default suite and CI do not install it, so this +# spec runs only where the operator has. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('nokogiri') + require 'filter/full_feed' require 'fileutils' require 'tmpdir' diff --git a/spec/plugins/filter/image_source_spec.rb b/spec/plugins/filter/image_source_spec.rb index c93e0e1..132f762 100644 --- a/spec/plugins/filter/image_source_spec.rb +++ b/spec/plugins/filter/image_source_spec.rb @@ -10,6 +10,11 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') +# FilterImageSource reads HTML with nokogiri, which the Gemfile declares in its +# optional :plugins group. The default suite and CI do not install it, so this +# spec runs only where the operator has. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('nokogiri') + require 'filter/image_source' describe Automatic::Plugin::FilterImageSource do diff --git a/spec/plugins/publish/markdown_spec.rb b/spec/plugins/publish/markdown_spec.rb index 54793b8..4bd2049 100644 --- a/spec/plugins/publish/markdown_spec.rb +++ b/spec/plugins/publish/markdown_spec.rb @@ -118,6 +118,52 @@ def publish(config, pipeline) end end + # nokogiri is an optional dependency: the plugin uses it where it is + # installed and reduces the body itself where it is not, so that a plain + # `gem install automatic` can run the Quick Start. See doc/PLUGINS.md + # section 6.7. The default suite runs the substitute, this run of it is + # explicit, and where the optional gem is installed the last example holds + # the two to the same answer. + describe 'a body reduced without an HTML parser' do + def publish_without_parser(config, pipeline) + output = StringIO.new + plugin = Automatic::Plugin::PublishMarkdown.new(config, pipeline) + plugin.instance_variable_set(:@output, output) + plugin.instance_variable_set(:@html_parser, false) + plugin.run + output.string + end + + before do + @pipeline = AutomaticSpec.generate_pipeline { + feed { + item 'https://example.com/a', 'A title', + "
\n

First & second.

\n" \ + " \n

Third
fourth

\n" \ + " \n

Fifth — sixth

\n
" + } + } + end + + it 'reduces the markup to text' do + document = publish_without_parser({}, @pipeline) + document.should include("First & second.\n\nThird\nfourth\n") + document.should_not include('

') + document.should_not include('alert(1)') + document.should_not include('a comment') + end + + it 'decodes a named entity beyond the five of XML' do + publish_without_parser({}, @pipeline).should include('Fifth — sixth') + end + + if AutomaticSpec.optional_dependency?('nokogiri') + it 'reduces it to what the parser reduces it to' do + publish_without_parser({}, @pipeline).should == publish({}, @pipeline) + end + end + end + describe 'content_encoded' do before do @pipeline = AutomaticSpec.generate_pipeline { diff --git a/spec/plugins/store/full_text_spec.rb b/spec/plugins/store/full_text_spec.rb index eeedafc..a1f307a 100644 --- a/spec/plugins/store/full_text_spec.rb +++ b/spec/plugins/store/full_text_spec.rb @@ -10,6 +10,13 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') +# The store plugins keep their records in SQLite through ActiveRecord. Both +# gems are the store plugins' own, declared in the Gemfile's optional :plugins +# and :store groups, so this spec runs only where the operator has installed +# them. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('activerecord') && + AutomaticSpec.optional_dependency?('sqlite3') + require 'store/full_text' require 'pathname' diff --git a/spec/plugins/store/permalink_spec.rb b/spec/plugins/store/permalink_spec.rb index a9664e4..cd930ad 100644 --- a/spec/plugins/store/permalink_spec.rb +++ b/spec/plugins/store/permalink_spec.rb @@ -10,6 +10,13 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') +# The store plugins keep their records in SQLite through ActiveRecord. Both +# gems are the store plugins' own, declared in the Gemfile's optional :plugins +# and :store groups, so this spec runs only where the operator has installed +# them. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('activerecord') && + AutomaticSpec.optional_dependency?('sqlite3') + require 'store/permalink' require 'pathname' diff --git a/spec/plugins/subscription/twitter_spec.rb b/spec/plugins/subscription/twitter_spec.rb index 1c19704..85b36d7 100644 --- a/spec/plugins/subscription/twitter_spec.rb +++ b/spec/plugins/subscription/twitter_spec.rb @@ -10,6 +10,11 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') +# SubscriptionTwitter reads HTML with nokogiri, which the Gemfile declares in +# its optional :plugins group. The default suite and CI do not install it, so +# this spec runs only where the operator has. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('nokogiri') + require 'subscription/twitter' describe Automatic::Plugin::SubscriptionTwitter do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c6eed05..f9b73cd 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -91,7 +91,12 @@ module AutomaticSpec # This is a declared list rather than something inferred from a load failure: # what the default suite does not cover is decided here, in one place, and a # spec that names a gem absent from this list is a mistake and says so. - OPTIONAL_PLUGIN_GEMS = %w[nkf sanitize].freeze + # + # The gems of the Supported (external) plugins are not here. Those plugins + # need a service or a command as well as a gem, they are outside the + # :plugins group for that reason, and their specs ask #plugin_available? + # instead. + OPTIONAL_PLUGIN_GEMS = %w[activerecord feedbag nkf nokogiri sanitize sqlite3].freeze class << self # Load a plugin, or report that its dependency is absent. @@ -120,7 +125,8 @@ def plugin_available?(path) # # Installing the group is what runs these: # - # BUNDLE_WITH=plugins bundle install + # bundle config set --local with plugins + # bundle install def optional_dependency?(gem_name) unless OPTIONAL_PLUGIN_GEMS.include?(gem_name) raise ArgumentError, @@ -131,7 +137,8 @@ def optional_dependency?(gem_name) skipped_optional << gem_name warn "[automatic] not verified by this run: the optional plugin gem " \ - "#{gem_name} is not installed (BUNDLE_WITH=plugins bundle install)" + "#{gem_name} is not installed " \ + '(bundle config set --local with plugins && bundle install)' false end