Make optional plugin gems truly optional with require_optional

.github/workflows/ci.yml

@@ -11,8 +11,13 @@ name: CI
 #
 # No credential is configured here and nothing reaches an external service: the
 # suite must need neither (doc/POLICY.md Invariant 6). Examples tagged :network
-# are excluded by default, and the Gemfile's optional :plugins group is not
+# are excluded by default, and none of the Gemfile's optional groups is
 # installed, so no plugin's own gem is a condition of this workflow passing.
+#
+# This is the required check, and it is deliberately the minimal configuration:
+# what it proves on every commit is that the framework needs nothing but its
+# four runtime dependencies. The all-plugins configuration is a separate,
+# non-required workflow, plugins.yml.
 
 on:
   push:

.github/workflows/plugins.yml

@@ -0,0 +1,55 @@
+name: Optional plugin dependencies
+
+# Installs the Gemfile's optional `plugins` group and runs the same suite, so
+# that the second documented way of setting up a checkout is checked as well as
+# described: the group resolves, and the plugin specs it brings in pass.
+#
+# This is not the required check. ci.yml is, and it installs none of this: what
+# has to hold on every commit is that the framework runs on its own runtime
+# dependencies (doc/POLICY.md sections 9.1 and 11). This workflow is the other
+# half of that statement, and it is held to the same standard — a failure here
+# is a real failure, to be fixed rather than silenced.
+#
+# One Ruby version, not the matrix. This checks the dependency configuration,
+# which the required workflow already checks across versions without it.
+#
+# Still no credential and no external service: the optional gems of the plugins
+# that need a running service are in their own groups, outside `plugins`, and
+# are not installed here.
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  plugins:
+    name: All supported optional plugin dependencies
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: Select the optional plugin group
+        run: bundle config set --local with plugins
+
+      - name: Install the bundle, including the optional plugin group
+        run: bundle install --jobs 4
+
+      - name: Check that the optional gems are in the bundle
+        run: |
+          bundle exec ruby -e "require 'nokogiri'; require 'active_record'; \
+            require 'sqlite3'; require 'sanitize'; require 'feedbag'; \
+            puts 'optional plugin dependencies present'"
+
+      - name: Run the test suite
+        run: bundle exec rake spec

Gemfile

@@ -2,6 +2,10 @@
 # The runtime and development dependencies are declared in automatic.gemspec,
 # which this file evaluates. Only the optional, plugin-specific gems are listed
 # here. See doc/POLICY.md section 9.
+#
+# `bundle install` with no configuration installs the framework's four runtime
+# dependencies and the development ones, and nothing below: a checkout is set
+# up to run and to test the framework, not to run every plugin.
 
 source 'https://rubygems.org'
 
@@ -11,23 +15,72 @@ gemspec
 # runtime dependencies of the gem: installing automatic does not install them,
 # and a Recipe that does not use the plugin does not need them.
 #
-# The group is optional, so `bundle install` does not install it and neither
-# the default test suite nor CI depends on it. Install it deliberately, and the
-# specs of the plugins that need it then run as part of the ordinary suite:
+# Every group here is optional, so nothing below is installed by default and
+# neither the default test suite nor required CI depends on any of it. Each gem
+# is in two groups: `plugins`, which is all of them at once, and one named
+# after what it is for, which is one of them on its own. Both are Bundler
+# groups and both are selected the same way, in the checkout's own .bundle
+# directory, which is not committed:
+#
+#   bundle config set --local with plugins     # all of the below
+#   bundle config set --local with store       # activerecord and sqlite3 only
+#   bundle config set --local with "store html"
+#   bundle install
 #
-#   BUNDLE_WITH=plugins bundle install
-#   bundle exec rake
+# Setting it in the configuration rather than passing BUNDLE_WITH to one
+# command is what makes the gems visible to `bundle exec` afterwards, so the
+# specs of the plugins that need them then run as part of the ordinary suite.
+# `bundle config unset --local with` returns the checkout to the minimum.
 #
 # The table of which plugin needs which gem, and which of those plugins still
 # work, is in doc/DEPLOYMENT.md and doc/PLUGINS.md section 6.
-group :plugins, optional: true do
-  gem 'nkf'               # FilterDescriptionLink
-  gem 'sanitize'          # FilterSanitize
 
-  # PublishAmazonS3 and the s3n:// path of StoreFile call AWS::S3, which only
-  # AWS SDK for Ruby v1 provided. No currently published gem satisfies them, so
-  # there is nothing to uncomment; they need rework. See doc/PLUGINS.md.
-  # gem 'dalli'           # PublishMemcached
-  # gem 'fluent-logger'   # PublishFluentd and ProvideFluentd
-  # gem 'xml-simple'      # CustomFeedSVNLog
+# StorePermalink and StoreFullText, through plugins/store/database.rb.
+group :plugins, :store, optional: true do
+  gem 'activerecord', '>= 7.1', '< 9.0'
+  gem 'sqlite3',      '>= 1.7', '< 3.0'
 end
+
+# An HTML parser, for the plugins that read HTML: FilterFullFeed,
+# FilterImageSource, FilterDescriptionLink, and FeedParser.parse_html for
+# SubscriptionLink and SubscriptionTumblr. PublishMarkdown uses it when it is
+# installed and reduces a body to text without it.
+group :plugins, :html, optional: true do
+  gem 'nokogiri', '>= 1.15', '< 2.0'
+end
+
+# FilterSanitize.
+group :plugins, :sanitize, optional: true do
+  gem 'sanitize'
+end
+
+# FilterDescriptionLink, which normalizes a fetched page's encoding with it.
+group :plugins, :nkf, optional: true do
+  gem 'nkf'
+end
+
+# The autodiscovery and inspect subcommands. No plugin and no Recipe uses it.
+group :plugins, :autodiscovery, optional: true do
+  gem 'feedbag', '>= 1.0', '< 2.0'
+end
+
+# The plugins below are Supported (external): each needs a service or a command
+# the operator provides, and their specs exercise it rather than a double. They
+# are deliberately outside the `plugins` group, so that installing that group
+# leaves the suite runnable with nothing else set up. Select one of these by
+# its own name when you have what it talks to.
+group :memcached, optional: true do
+  gem 'dalli'             # PublishMemcached, with a memcached server
+end
+
+group :fluentd, optional: true do
+  gem 'fluent-logger'     # PublishFluentd and ProvideFluentd, with a Fluentd instance
+end
+
+group :svn_log, optional: true do
+  gem 'xml-simple'        # CustomFeedSVNLog, with the svn command
+end
+
+# PublishAmazonS3 and the s3n:// path of StoreFile call AWS::S3, which only
+# AWS SDK for Ruby v1 provided. No currently published gem satisfies them, so
+# there is no group to select; they need rework. See doc/PLUGINS.md.

README.md

@@ -110,11 +110,14 @@ plainly which of its plugins still work. See [`doc/VERSIONS`](doc/VERSIONS).
 - **Your plugins override the shipped ones.** `~/.automatic/plugins` is searched
   first, so a shipped plugin can be replaced without touching the installation.
 - **De-duplication built in.** The store plugins keep a SQLite record of what
-  has been seen, which is what makes a Recipe safe to run every hour.
+  has been seen, which is what makes a Recipe safe to run every hour. Their
+  gems are installed when you use them, not before.
 - **Retry and interval** on everything that reaches the network, configured per
   plugin in the Recipe.
 - **A small installation.** A gem needed by one plugin is not a dependency of
-  the framework, so installing this does not install an AWS SDK.
+  the framework: `gem install automatic` brings four pure-Ruby gems and the
+  command, and installs neither an HTML parser nor a database — let alone an
+  AWS SDK.
 - **Honest about what is broken.** Every plugin is classified, with its reason,
   in [`doc/PLUGINS.md`](doc/PLUGINS.md). Nothing dead is stubbed into looking
   alive.
@@ -163,7 +166,9 @@ The full account is [`doc/BASIC_DESIGN.md`](doc/BASIC_DESIGN.md).
 - **Ruby 3.3 through 4.0.** CI validates 3.3, 3.4 and 4.0.
 - A Unix-like system. GNU/Linux and macOS are what it is used on. Windows is not
   supported.
-- A compiler, if `nokogiri` or `sqlite3` build from source on your platform.
+- A compiler only if you install an optional plugin gem that builds from source
+  on your platform, such as `nokogiri` or `sqlite3`. The framework's own
+  dependencies are pure Ruby.
 
 Ruby 3.3 is the floor: it is the oldest maintained release the dependencies are
 resolved and tested against. Nothing older is tested or supported.
@@ -187,33 +192,49 @@ gem install automatic
 automatic --version
 ```
 
+That installs the framework, the command and four pure-Ruby dependencies.
+A gem that only one plugin needs is not among them: install it when you use
+that plugin, with `gem install nokogiri` or `gem install activerecord sqlite3`.
+[`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md) lists which plugin needs which.
+
 ### From a checkout
 
 Use a checkout to try the current development version, change the source,
-develop a plugin or verify changes before a release:
+develop a plugin or verify changes before a release. There are three ways to
+set one up; start with the first.
 
 ```sh
 git clone https://github.com/id774/automaticruby.git
 cd automaticruby
+
+# Minimal: the framework and its test suite. No optional plugin gem.
+bundle install
+
+# All supported optional plugin dependencies, for plugin work.
+bundle config set --local with plugins
+bundle install
+
+# Or start minimal and add one group at a time, as you use its plugins.
+bundle config set --local with store
 bundle install
+```
+
+```sh
 bundle exec bin/automatic --version
 bundle exec rake
 ```
 
-`bundle install` resolves the runtime and development dependencies declared by
-`Gemfile` and `automatic.gemspec`, and installs the gems needed to run and test
-the checkout. If the `bundle` command is unavailable, install Bundler first
-with `gem install bundler`.
-
-In a checkout, every `automatic` below becomes `bundle exec bin/automatic`.
-Use `bundle exec rake` to verify the development environment by running the
-test suite.
+A plain `bundle install` resolves the runtime dependencies declared by
+`automatic.gemspec` and the development ones, and installs no optional plugin
+gem: those are optional Bundler groups, which are installed only when asked
+for. If the `bundle` command is unavailable, install Bundler first with
+`gem install bundler`.
 
-The core development setup does not install the optional `plugins` bundle
-group. Some plugins need their own gem or external service; install those only
-for the plugin being used or developed. Check the plugin catalogue in
-[`doc/PLUGINS.md`](doc/PLUGINS.md) and the dependency table in
-[`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md).
+In a checkout, every `automatic` below becomes `bundle exec bin/automatic`, and
+`bundle exec` sees only the bundle — so a plugin's gem is added with a group
+rather than with `gem install`. The group names, and which plugin needs which
+gem, are in [`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md); what each plugin does is
+in [`doc/PLUGINS.md`](doc/PLUGINS.md).
 
 ## 6. Quick start
 
@@ -228,11 +249,13 @@ automatic -c ~/.automatic/config/example/feed2markdown.yml
 `assets/`, and copies the example Recipes into `~/.automatic/config/example`.
 It never overwrites anything already there.
 
-That Recipe fetches the public Ruby news feed, skips what it has published
-before, and appends the rest to `~/.automatic/markdown/feeds.md`. Run it twice: the second
-run leaves the file alone, because the store plugin has seen it all. Read the
-file, `grep` it, put it in a repository, or hand it to whatever reads text next.
-`feed2console.yml` beside it is the same pipeline printing to the terminal.
+That Recipe fetches the public Ruby news feed and appends its items to
+`~/.automatic/markdown/feeds.md`, using nothing but the framework and what
+`gem install automatic` brought. Read the file, `grep` it, put it in a
+repository, or hand it to whatever reads text next. `feed2console.yml` beside
+it is the same pipeline printing to the terminal. Adding a store plugin, so
+that a second run appends only what is new, is step 5 of the Quick Start and
+the point at which the first optional gems are installed.
 
 To check the framework without any network, write this instead:
 
@@ -485,12 +508,13 @@ COVERAGE=on bundle exec rake spec
   AUTOMATIC_NETWORK_SPECS=1 bundle exec rake spec
   ```
 
-- A plugin whose gem the Gemfile declares in its optional `:plugins` group is
-  **not verified by the default suite**, because that group is not installed.
-  Install it to run those specs as part of the ordinary suite:
+- A plugin whose gem the Gemfile declares in an optional group is **not
+  verified by the default suite**, because no optional group is installed.
+  Install them to run those specs as part of the ordinary suite:
 
   ```sh
-  BUNDLE_WITH=plugins bundle install
+  bundle config set --local with plugins
+  bundle install
   bundle exec rake
   ```
 
@@ -502,11 +526,16 @@ COVERAGE=on bundle exec rake spec
   in CI. Most need a credential, a dead service, or both — read one before
   running it.
 
-CI installs the bundle, builds the gem, loads the library, runs the CLI and runs
-the default suite on each validated Ruby version, from
+The required check installs the bundle, builds the gem, loads the library, runs
+the CLI and runs the default suite on each validated Ruby version, from
 [`.github/workflows/ci.yml`](.github/workflows/ci.yml). It configures no secret
 and installs no optional plugin gem, so no plugin's own dependency is a
-condition of a green build.
+condition of a change being merged — and what it proves on every commit is that
+the framework needs nothing but its own runtime dependencies. A separate,
+non-required workflow,
+[`.github/workflows/plugins.yml`](.github/workflows/plugins.yml), installs the
+`plugins` group and runs the same suite, which is how the all-plugins setup is
+checked.
 
 ## 13. Development
 

automatic.gemspec

@@ -86,31 +86,26 @@ Gem::Specification.new do |spec|
   spec.require_paths      = ['lib']
   spec.extra_rdoc_files   = ['README.md', 'doc/LICENSE.md']
 
-  # Runtime dependencies: what the framework itself needs, plus what the
-  # documented primary workflow needs — the store plugins, which nearly every
-  # Recipe uses to avoid repeating its work, and the Markdown publisher.
+  # Runtime dependencies: what the framework in lib/ requires, and nothing
+  # else. Requiring `automatic`, loading a Recipe, loading a plugin, running a
+  # pipeline and the CLI's own work are what these four are for.
   #
-  # A gem needed by a single plugin is NOT declared here. It is required inside
-  # that plugin's own file and installed by the operator who uses the plugin.
-  # See doc/POLICY.md section 9.1 and doc/DEPLOYMENT.md.
+  # A gem needed by a plugin is NOT declared here, however useful that plugin
+  # is. It is required inside the plugin's own file and installed by the
+  # operator who uses the plugin: `gem install automatic` therefore installs no
+  # HTML parser, no database and no service client. The optional gems, which
+  # plugin needs which, and how to install them are in the Gemfile's optional
+  # groups, doc/DEPLOYMENT.md and doc/POLICY.md section 9.1.
   #
   # rexml and rss left the standard library and became gems over the 3.x
   # series, and nkf followed after 3.3. Each gem listed here is listed because
-  # something committed here requires it, and a library's move out of the
-  # standard library is not by itself a reason to declare it: nkf is a plugin's
-  # dependency and is in the Gemfile's optional group instead.
-  spec.add_dependency 'activerecord',  '>= 7.1', '< 9.0'   # store plugins
+  # a file in lib/ requires it, and a library's move out of the standard
+  # library is not by itself a reason to declare it: nkf is a plugin's
+  # dependency and is in the Gemfile's optional groups instead.
   spec.add_dependency 'activesupport', '>= 7.1', '< 9.0'   # plugin loader, XML subscription
-  spec.add_dependency 'feedbag',       '>= 1.0', '< 2.0'   # autodiscovery subcommand
   spec.add_dependency 'hashie',        '>= 4.0', '< 6.0'   # Recipe
-  # Used by no framework file on the way in: requiring `automatic` loads no
-  # HTML parser. It is here because Supported plugins that an installed gem
-  # must be able to run need it -- PublishMarkdown, and FeedParser.parse_html
-  # for SubscriptionLink and SubscriptionTumblr.
-  spec.add_dependency 'nokogiri',      '>= 1.15', '< 2.0'  # HTML parsing, in plugins
   spec.add_dependency 'rexml',         '>= 3.2', '< 4.0'   # OPML parser
   spec.add_dependency 'rss',           '>= 0.3', '< 1.0'   # the pipeline value
-  spec.add_dependency 'sqlite3',       '>= 1.7', '< 3.0'   # store plugins
 
   spec.add_development_dependency 'rake',                      '~> 13.0'
   spec.add_development_dependency 'rspec',                     '~> 3.13'