diff --git a/.gitignore b/.gitignore index aac9f85ba5..042c2981a6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ tmp public/assets public/fonts public/docs/**/* +public/index.html +public/service-worker.js +public/feed.atom docs/**/* !docs/*.md /vendor diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000000..cc1c3b4d1e --- /dev/null +++ b/Caddyfile @@ -0,0 +1,185 @@ +# Caddyfile — serve DevDocs as static files, with no Ruby at request time. +# +# Serves the prerendered output of `rake assets:precompile` +# (public/index.html, public/service-worker.js, public/feed.atom) plus the +# compiled assets under public/. This is an ALTERNATIVE production front end; +# the Sinatra app (config.ru) is untouched and remains the dev/preview server +# and the source of truth for the redirect/header rules mirrored below. +# +# Local preview (no ACME, single host): +# DEVDOCS_SITE_ADDRESS=:8080 caddy run --config Caddyfile +# # then comment out the `out.devdocs.io` block below +# Production: +# caddy run --config Caddyfile # automatic HTTPS for devdocs.io + +# --------------------------------------------------------------------------- +# Outbound sponsor / tracking links (302). Shared by both hostnames because +# out.devdocs.io keeps serving /s/* while redirecting everything else. +# Mirrors the `/s/*` hash in lib/app.rb. +# --------------------------------------------------------------------------- +(sponsors) { + redir /s/maxcdn https://www.maxcdn.com/?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs + redir /s/shopify https://www.shopify.com/careers?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs + redir /s/jetbrains https://www.jetbrains.com/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs + redir /s/jetbrains/ruby https://www.jetbrains.com/ruby/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs + redir /s/jetbrains/python https://www.jetbrains.com/pycharm/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs + redir /s/jetbrains/c https://www.jetbrains.com/clion/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs + redir /s/jetbrains/web https://www.jetbrains.com/webstorm/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs + redir /s/code-school https://www.codeschool.com/?utm_campaign=devdocs&utm_content=homepage&utm_source=devdocs&utm_medium=sponsorship + redir /s/tw "https://twitter.com/intent/tweet?url=http%3A%2F%2Fdevdocs.io&via=DevDocs&text=All-in-one%20API%20documentation%20browser%20with%20offline%20mode%20and%20instant%20search%3A" + redir /s/fb "https://www.facebook.com/sharer/sharer.php?u=http%3A%2F%2Fdevdocs.io" + redir /s/re "https://www.reddit.com/submit?url=http%3A%2F%2Fdevdocs.io&title=All-in-one%20API%20documentation%20browser%20with%20offline%20mode%20and%20instant%20search&resubmit=true" +} + +# --------------------------------------------------------------------------- +# Main site +# --------------------------------------------------------------------------- +{$DEVDOCS_SITE_ADDRESS:devdocs.io} { + root * {$DEVDOCS_ROOT:./public} + encode zstd gzip + + # --- Security headers (mirrors lib/app.rb production :csp + SslEnforcer hsts) --- + header { + Content-Security-Policy "default-src 'self' *; script-src 'self' 'nonce-devdocs' https://www.google-analytics.com https://secure.gaug.es https://*.jquery.com; font-src 'none'; style-src 'self' 'unsafe-inline' *; img-src 'self' * data:;" + Strict-Transport-Security "max-age=31536000; includeSubDomains" + -Server + } + + # --- Cache-Control (mirrors Rack::Static header_rules) --- + @assets path /assets/* + header @assets Cache-Control "public, max-age=604800" + @day path /docs/* /images/* /favicon.ico /robots.txt /opensearch.xml /mathml.css /manifest.json + header @day Cache-Control "public, max-age=86400" + @nocache not path /assets/* /docs/* /images/* /favicon.ico /robots.txt /opensearch.xml /mathml.css /manifest.json + header @nocache Cache-Control "no-cache, max-age=0" + + # All routing in one ordered block: first matching terminal handler wins. + route { + import sponsors + + # --- Gone (410) — mirrors the /maxcdn routes --- + @maxcdn path /maxcdn /maxcdn/ + respond @maxcdn 410 + + # --- Health check --- + respond /ping 200 + + # --- Search → client-side hash query (/search?q=… and /?q=…) --- + redir /search /#q={query.q} + @rootq { + path / + query q=* + } + redir @rootq /#q={query.q} + + # --- Legacy path redirects (301) --- + redir /tips /help permanent + redir /css-data-types/ /css-values-units/ permanent + redir /css-at-rules/ "/?q=css%20%40" permanent + redir /dom/window/setinterval /dom/windoworworkerglobalscope/setinterval permanent + redir /html/article /html/element/article permanent + redir /html-html5/ /html-elements/ permanent + redir /html-standard/ /html-elements/ permanent + redir /http-status-codes/ /http-status/ permanent + redir /ruby/bignum /ruby~2.3/bignum permanent + redir /ruby/fixnum /ruby~2.3/fixnum permanent + + # --- Documentation renames (301, path-preserving) — mirrors DOC_REDIRECTS --- + @r_iojs path_regexp iojs ^/iojs([/-].*)?$ + redir @r_iojs /node{re.iojs.1} permanent + @r_node_lts path_regexp node_lts ^/node_lts([/-].*)?$ + redir @r_node_lts /node~6_lts{re.node_lts.1} permanent + @r_node42 path_regexp node42 ^/node~4\.2_lts([/-].*)?$ + redir @r_node42 /node~4_lts{re.node42.1} permanent + @r_yii1 path_regexp yii1 ^/yii1([/-].*)?$ + redir @r_yii1 /yii~1.1{re.yii1.1} permanent + @r_python2 path_regexp python2 ^/python2([/-].*)?$ + redir @r_python2 /python~2.7{re.python2.1} permanent + @r_xpath path_regexp xpath ^/xpath([/-].*)?$ + redir @r_xpath /xslt_xpath{re.xpath.1} permanent + @r_ng4ts path_regexp ng4ts ^/angular~4_typescript([/-].*)?$ + redir @r_ng4ts /angular{re.ng4ts.1} permanent + @r_ng2ts path_regexp ng2ts ^/angular~2_typescript([/-].*)?$ + redir @r_ng2ts /angular~2{re.ng2ts.1} permanent + @r_ng20ts path_regexp ng20ts ^/angular~2\.0_typescript([/-].*)?$ + redir @r_ng20ts /angular~2{re.ng20ts.1} permanent + @r_ng15 path_regexp ng15 ^/angular~1\.5([/-].*)?$ + redir @r_ng15 /angularjs~1.5{re.ng15.1} permanent + @r_ng14 path_regexp ng14 ^/angular~1\.4([/-].*)?$ + redir @r_ng14 /angularjs~1.4{re.ng14.1} permanent + @r_ng13 path_regexp ng13 ^/angular~1\.3([/-].*)?$ + redir @r_ng13 /angularjs~1.3{re.ng13.1} permanent + @r_ng12 path_regexp ng12 ^/angular~1\.2([/-].*)?$ + redir @r_ng12 /angularjs~1.2{re.ng12.1} permanent + @r_ci30 path_regexp ci30 ^/codeigniter~3\.0([/-].*)?$ + redir @r_ci30 /codeigniter~3{re.ci30.1} permanent + @r_pt1 path_regexp pt1 ^/pytorch~1([/-].*)?$ + redir @r_pt1 /pytorch~1.13{re.pt1.1} permanent + @r_pt2 path_regexp pt2 ^/pytorch~2([/-].*)?$ + redir @r_pt2 /pytorch{re.pt2.1} permanent + @r_wp2 path_regexp wp2 ^/webpack~2([/-].*)?$ + redir @r_wp2 /webpack{re.wp2.1} permanent + + # --- MDN angular/dom fragment rewrites (301) — mirrors the catch-all --- + # The regexes are mutually exclusive, so their order is irrelevant. + @ng_api path_regexp ng_api ^/angular(/ng.*)$ + redir @ng_api /angularjs/api{re.ng_api.1} permanent + @dom_wt path_regexp dom_wt ^/dom/windowtimers(.*)$ + redir @dom_wt /dom/windoworworkerglobalscope{re.dom_wt.1} permanent + @dom_url path_regexp dom_url ^/dom/window/url\.(.*)$ + redir @dom_url /dom/url/{re.dom_url.1} permanent + @dom_win path_regexp dom_win ^/dom/window\.(.*)$ + redir @dom_win /dom/window/{re.dom_win.1} permanent + @dom_el path_regexp dom_el ^/dom/element\.(.*)$ + redir @dom_el /dom/element/{re.dom_el.1} permanent + @dom_ev path_regexp dom_ev ^/dom/event\.(.*)$ + redir @dom_ev /dom/event/{re.dom_ev.1} permanent + @dom_doc path_regexp dom_doc ^/dom/document\.(.*)$ + redir @dom_doc /dom/document/{re.dom_doc.1} permanent + + # --- News feed (static Atom at /feed and /feed.atom) --- + # Terminal handle, before canonicalization, so /feed isn't treated as a + # bare doc slug and given a trailing slash. + @feed path /feed /feed.atom + handle @feed { + rewrite * /feed.atom + file_server + } + + # --- Documentation URL canonicalization (mirrors the catch-all) --- + # Strip a trailing slash from entry paths (two+ segments): /css/x/ -> /css/x + @entry_slash path_regexp entry_slash ^(/[^/]+/.+)/$ + redir @entry_slash {re.entry_slash.1} permanent + # Add a trailing slash to a bare doc slug: /css -> /css/. Excludes the app + # pages and the static root files, which are served as-is below. + @doc_index { + path_regexp doc_index ^/([\w.~%-]+)$ + not path /settings /offline /about /news /help + not path /favicon.ico /robots.txt /opensearch.xml /mathml.css /manifest.json /service-worker.js + } + redir @doc_index {path}/ permanent + + # --- Static files, with SPA fallback to the prerendered shell --- + # Real files (assets, docs, images, service-worker.js, feed.atom, the app + # pages) are served from disk; every other path falls back to index.html so + # the client-side router resolves it. + try_files {path} /index.html + file_server + } + + # 404/500 → the prebuilt error pages (mirrors not_found/error in app.rb). + handle_errors { + rewrite * /{err.status_code}.html + file_server + } +} + +# --------------------------------------------------------------------------- +# Tracking domain: out.devdocs.io keeps serving /s/* and 302-redirects +# everything else to the canonical host. Mirrors the OUT_HOST `before` filter. +# Remove this block for local preview. +# --------------------------------------------------------------------------- +out.devdocs.io { + import sponsors + redir * https://devdocs.io{uri} +} diff --git a/Rakefile b/Rakefile index 705c6ec1ba..a74e3cfa4e 100644 --- a/Rakefile +++ b/Rakefile @@ -19,5 +19,6 @@ namespace :assets do load 'tasks/assets.thor' AssetsCLI.new.compile + AssetsCLI.new.render_static end end diff --git a/assets/javascripts/app/app.js b/assets/javascripts/app/app.js index 3ced2c403c..9d0729b4e5 100644 --- a/assets/javascripts/app/app.js +++ b/assets/javascripts/app/app.js @@ -8,6 +8,11 @@ class App extends Events { views = {}; init() { + // Determine the boot mode from the URL before anything else, so error + // tracking and the rest of init can rely on isSingleDoc(). + const singleDoc = this.singleDocFromLocation(); + this.singleDoc = !!singleDoc; + try { this.initErrorTracking(); } catch (error) {} @@ -36,8 +41,8 @@ class App extends Events { this.mobile = new app.views.Mobile(); } - if (document.body.hasAttribute("data-doc")) { - this.DOC = JSON.parse(document.body.getAttribute("data-doc")); + if (singleDoc) { + this.DOC = singleDoc; this.bootOne(); } else if (this.DOCS) { this.bootAll(); @@ -46,6 +51,26 @@ class App extends Events { } } + // Detects whether the current URL is a direct link to a single documentation + // (e.g. /python~3.12/functions) and, if so, returns its entry from the + // catalog. Anything else — the root app or a path that isn't a doc slug (the + // app pages, unknown docs) — returns undefined and boots the full app, which + // resolves aliases/redirects. + singleDocFromLocation() { + if (!this.DOCS) { + return; + } + const segment = location.pathname.split("/")[1]; + if (!segment) { + return; + } + const slug = decodeURIComponent(segment); + return ( + this.DOCS.find((doc) => doc.slug === slug) || + this.DOCS.find((doc) => doc.slug.split("~")[0] === slug) + ); + } + browserCheck() { if (this.isSupportedBrowser()) { return true; @@ -115,6 +140,7 @@ class App extends Events { }); new app.views.Notice("singleDoc", this.doc); delete this.DOC; + delete this.DOCS; } bootAll() { @@ -383,7 +409,7 @@ Please check your browser extensions/addons. `); } isSingleDoc() { - return document.body.hasAttribute("data-doc"); + return !!this.singleDoc; } isMobile() { diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000000..68a0820d2d --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,83 @@ +# Client-Side Storage + +DevDocs is a client-side app: the Sinatra server only serves the app shell, +static assets, and the scraped documentation files. Which docs a user selected +and what is available offline lives entirely in the browser, across four +independent layers. Notably, the server does **not** read any of this to render +the app. + +## 1. Enabled-docs list → cookie (`docs`) + +The selected doc slugs, joined by `/` (e.g. `css/html/javascript~5`). + +- **Where:** a browser cookie `docs` (the settings store is a `CookiesStore`). +- **Read/written:** `Settings#getDocs` (falls back to `default_docs`) / + `Settings#setDocs`, toggled in the Settings page + ([`app/settings.js`](../assets/javascripts/app/settings.js)). +- **Used at boot:** `App#bootAll` splits the full catalog into `app.docs` + (enabled) and `app.disabledDocs` ([`app/app.js`](../assets/javascripts/app/app.js)). + +Only this item is a cookie rather than `localStorage`, and it no longer needs to +be — nothing server-side reads it. + +## 2. Per-doc index (`index.json`) → localStorage + +The table of contents (entries + types) that powers the sidebar and search — not +the page content. In [`models/doc.js`](../assets/javascripts/models/doc.js): + +- **Where:** `app.localStorage` (`LocalStorageStore`), keyed by doc slug, value + `[mtime, data]`. +- **Read:** `Doc#_getCache` returns data only if the stored `mtime` matches; + otherwise it clears the stale entry. +- **Written / updated:** `Doc#load({readCache, writeCache})` reads `localStorage` + first (a hit ⇒ no network); on a miss (first enable or changed `mtime`) it + fetches `index.json` from `docs_origin` — the docs CDN, a different origin in + production — and writes it back. Runs for every enabled doc at boot via + `Docs#load`. + +## 3. Doc content (page HTML) → IndexedDB, or fetched live + +Two modes, in [`app/db.js`](../assets/javascripts/app/db.js): + +- **Not installed (default):** fetched **per page, on demand** — + `DB#loadWithXHR` → `entry.fileUrl()`. Not stored. +- **Installed for offline** (explicit download or `autoInstall`): `Doc#install` + fetches the whole `db.json`; `DB#store` writes every entry's HTML into + **IndexedDB** (one object store per slug, plus a `docs` store mapping slug → + `mtime`). +- **Read:** `DB#load` uses IndexedDB when the doc is installed, else falls back + to network. + +## 4. App shell + fetched index files → Service Worker Cache + +What makes the app work offline. Two parts with different justifications +([`views/service-worker.js.erb`](../views/service-worker.js.erb)): + +- **App shell (essential).** On install, precaches `/`, favicon, manifest, and + the fingerprinted JS/CSS/sprites. Offline, *something* must answer the request + for the HTML document and assets; `localStorage`/IndexedDB can't (they're read + only after the app is running). A service worker is the only primitive that can + serve navigations/assets with no network — this is why offline works at all. + The precache list is kept doc-independent so a flaky doc index can't fail the + atomic `cache.addAll` install. +- **Index files (fallback for layer 2's quota).** The `fetch` handler also caches + `index.json` at runtime (from the docs CDN, which sends CORS headers so the + response is cacheable regardless of origin). This seems redundant with layer 2, + but `localStorage` has a ~5 MB quota and `LocalStorageStore.set` swallows + `QuotaExceededError` silently — so with many/large docs, some indexes never + persist there and are re-fetched offline. Only the Cache API (far larger quota) + can then satisfy them, making the service worker the reliable high-capacity + index store, with `localStorage` as the fast first hit. + +## Summary + +| Layer | What | Storage | Written when | +| --- | --- | --- | --- | +| Enabled list | doc slugs | Cookie `docs` | toggled in Settings | +| Doc index | TOC / entries / types (`index.json`) | `localStorage`, `[mtime, data]` per slug | `Doc#load` fetch (enable / `mtime` change) | +| Doc content | page HTML | IndexedDB per slug (installed); else live XHR | `Doc#install` → `DB#store` | +| Shell + index copy | JS/CSS/sprites + `index.json` | Service Worker Cache | SW install (shell) + runtime fetch (index) | + +Layers 1–3 are client-managed and drive the running app. Layer 4 lets it boot +offline (the shell) and backstops layer 2 when the index exceeds `localStorage`'s +quota. diff --git a/lib/app.rb b/lib/app.rb index 3b59b526e1..29c3252111 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -5,7 +5,6 @@ class App < Sinatra::Application Bundler.require environment - require 'sinatra/cookies' require 'tilt/erubi' require 'active_support/notifications' @@ -131,13 +130,8 @@ def self.parse_news end helpers do - include Sinatra::Cookies include Sprockets::Helpers - def memoized_cookies - @memoized_cookies ||= cookies.to_hash - end - def canonical_origin "https://#{request.host_with_port}" end @@ -150,18 +144,6 @@ def unsupported_browser? browser.ie? end - def docs - @docs ||= begin - cookie = memoized_cookies['docs'] - - if cookie.nil? - settings.default_docs - else - cookie.split('/') - end - end - end - def find_doc(slug) settings.docs[slug] || begin settings.docs.each do |_, doc| @@ -171,25 +153,6 @@ def find_doc(slug) end end - def user_has_docs?(slug) - docs.include?(slug) || begin - slug = "#{slug}~" - docs.any? { |_slug| _slug.start_with?(slug) } - end - end - - def doc_index_urls - docs.each_with_object [] do |slug, result| - if doc = settings.docs[slug] - result << "#{settings.docs_origin}/#{slug}/index.json?#{doc['mtime']}" - end - end - end - - def doc_index_page? - @doc && (request.path == "/#{@doc['slug']}/" || request.path == "/#{@doc['slug_without_version']}/") - end - def query_string_for_redirection request.query_string.empty? ? nil : "?#{request.query_string}" end @@ -400,7 +363,7 @@ def service_worker_cache_name redirect "/#{doc}#{type}#{rest[0...-1]}#{query_string_for_redirection}" else response.headers['Content-Security-Policy'] = settings.csp if settings.csp - erb :other + erb :index end end diff --git a/lib/tasks/assets.thor b/lib/tasks/assets.thor index e9de1b6ffe..99175d6f8c 100644 --- a/lib/tasks/assets.thor +++ b/lib/tasks/assets.thor @@ -28,6 +28,39 @@ class AssetsCLI < Thor manifest.clean(options[:keep]) end + # Render the dynamic shells and the news feed to static files so the app can + # be served without the Ruby process. Rendered through the real Rack app so + # the output is identical to what Sinatra serves. Must run after `compile`. + desc 'render_static', 'Render index.html, service-worker.js and feed.atom to public/' + def render_static + require 'rack' + + # The app booted (in `initialize`) before `compile` wrote the asset + # manifest, so point the asset helpers at the freshly compiled manifest. + Sprockets::Helpers.configure do |config| + config.manifest = Sprockets::Manifest.new(nil, App.assets_manifest_path) + end + + origin = ENV['DEPLOY_HOST'] || 'devdocs.io' + + { + '/' => 'index.html', + '/service-worker.js' => 'service-worker.js', + '/feed' => 'feed.atom', + }.each do |path, filename| + env = Rack::MockRequest.env_for("https://#{origin}#{path}") + status, _headers, body = App.call(env) + raise "Failed to render #{path} (status #{status})" unless status == 200 + + content = +'' + body.each { |part| content << part } + body.close if body.respond_to?(:close) + + File.write(File.join(App.public_folder, filename), content) + logger.info("Rendered #{path} -> public/#{filename}") + end + end + private def sprockets diff --git a/test/app_test.rb b/test/app_test.rb index 4b5e4c3297..135cbe9f09 100644 --- a/test/app_test.rb +++ b/test/app_test.rb @@ -139,13 +139,11 @@ def app it "works when the doc exists" do get '/html~4-foo-bar_42/' assert last_response.ok? - assert_includes last_response.body, 'data-doc="{"name":"HTML","slug":"html~4"' end it "works when the doc has no version in the path and a version exists" do get '/html-foo-bar_42/' assert last_response.ok? - assert_includes last_response.body, 'data-doc="{"name":"HTML","slug":"html~5"' end it "returns 404 when the type is blank" do diff --git a/views/other.erb b/views/other.erb deleted file mode 100644 index abd8bc20e1..0000000000 --- a/views/other.erb +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - <% if doc_index_page? %><% else %><% end %> - - - - DevDocs<%= " — #{@doc['full_name']} documentation" if doc_index_page? %> - - - - - <%= stylesheet_tag 'application' %> - - - -<%= erb :app -%> -<%= javascript_tag 'application' %><% unless App.production? %> -<%= javascript_tag 'debug' %><% end %> - - diff --git a/views/service-worker.js.erb b/views/service-worker.js.erb index 81689a2efb..d6fe4b493c 100644 --- a/views/service-worker.js.erb +++ b/views/service-worker.js.erb @@ -8,7 +8,6 @@ const urlsToCache = [ '/favicon.ico', '/manifest.json', '<%= service_worker_asset_urls.join "',\n '" %>', - '<%= doc_index_urls.join "',\n '" %>', ]; <%# Set-up the cache %> @@ -37,6 +36,22 @@ self.addEventListener('fetch', event => { try { const response = await fetch(event.request); + + <%# Cache documentation index files as they are fetched, so that enabled %> + <%# docs stay available offline without baking the (per-user) list of %> + <%# enabled docs into this file at request time. index.json is served %> + <%# from the docs CDN (a different origin in production), which sends CORS %> + <%# headers, so the response is inspectable and cacheable — no origin check. %> + const url = new URL(event.request.url); + if ( + event.request.method === 'GET' && + url.pathname.endsWith('/index.json') && + response.ok + ) { + const cache = await caches.open(cacheName); + cache.put(event.request, response.clone()); + } + return response; } catch (err) { const url = new URL(event.request.url);