Send ETag and Last-Modified with rewritten npm and Composer metadata

docs/architecture.md

@@ -189,8 +189,19 @@ metadata_cache (
     updated_at    DATETIME
 )
 -- indexes: (ecosystem, name) unique
+
+metadata_dates (
+    ecosystem     TEXT NOT NULL,
+    proxy_url     TEXT NOT NULL,      -- each proxy URL rewrites to its own download URLs
+    name          TEXT NOT NULL,
+    etag          TEXT NOT NULL,      -- the rewritten representation being dated
+    last_modified INTEGER NOT NULL,   -- Unix seconds; BIGINT on Postgres
+    PRIMARY KEY (ecosystem, proxy_url, name)
+)
 ```
 
+`metadata_dates` holds the `Last-Modified` served with rewritten npm and Composer metadata. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have.
+
 On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`.
 
 The `MigrateSchema()` function handles backward compatibility with older git-pkgs databases by running named migrations that add missing columns and tables. See [migrations.md](migrations.md) for how to add new schema changes.

internal/database/database_test.go

@@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB {
 	// Drop and recreate every table CreateSchema creates for clean test state;
 	// leftover migration records make the next CreateSchema fail on the
 	// migrations primary key.
-	tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "migrations", "schema_info"}
+	tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "migrations", "schema_info"}
 	for _, table := range tables {
 		_, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE")
 	}

internal/database/metadata_dates.go

@@ -0,0 +1,77 @@
+package database
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+)
+
+// MetadataDate is the Last-Modified a package's rewritten metadata is served
+// with, and the ETag of the rewrite it dates.
+//
+// Rewritten metadata has no date of its own, so the proxy assigns one when a
+// package starts serving a new rewrite. Keeping it in the database rather than
+// in process memory is what lets replicas sharing the database agree on it: a
+// date one replica gave out is never given by another to a different rewrite.
+type MetadataDate struct {
+	ETag         string
+	LastModified time.Time
+}
+
+type metadataDateRow struct {
+	ETag         string `db:"etag"`
+	LastModified int64  `db:"last_modified"`
+}
+
+// GetMetadataDate returns the date recorded for a package's metadata as
+// rewritten for proxyURL, or nil when there is none.
+func (db *DB) GetMetadataDate(ecosystem, proxyURL, name string) (*MetadataDate, error) {
+	var row metadataDateRow
+	err := db.Get(&row, db.Rebind(`
+		SELECT etag, last_modified FROM metadata_dates
+		WHERE ecosystem = ? AND proxy_url = ? AND name = ?
+	`), ecosystem, proxyURL, name)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("getting metadata date: %w", err)
+	}
+	return &MetadataDate{ETag: row.ETag, LastModified: time.Unix(row.LastModified, 0).UTC()}, nil
+}
+
+// InsertMetadataDate records the first date of a package's metadata and
+// reports whether it did; it leaves a date another replica recorded first.
+func (db *DB) InsertMetadataDate(ecosystem, proxyURL, name string, date MetadataDate) (bool, error) {
+	result, err := db.Exec(db.Rebind(`
+		INSERT INTO metadata_dates (ecosystem, proxy_url, name, etag, last_modified)
+		VALUES (?, ?, ?, ?, ?)
+		ON CONFLICT (ecosystem, proxy_url, name) DO NOTHING
+	`), ecosystem, proxyURL, name, date.ETag, date.LastModified.Unix())
+	if err != nil {
+		return false, fmt.Errorf("inserting metadata date: %w", err)
+	}
+	return affectedOne(result)
+}
+
+// SwapMetadataDate replaces a package's date with next if it is still old,
+// and reports whether it did; it leaves a date another replica swapped in.
+func (db *DB) SwapMetadataDate(ecosystem, proxyURL, name string, old, next MetadataDate) (bool, error) {
+	result, err := db.Exec(db.Rebind(`
+		UPDATE metadata_dates SET etag = ?, last_modified = ?
+		WHERE ecosystem = ? AND proxy_url = ? AND name = ? AND etag = ? AND last_modified = ?
+	`), next.ETag, next.LastModified.Unix(), ecosystem, proxyURL, name, old.ETag, old.LastModified.Unix())
+	if err != nil {
+		return false, fmt.Errorf("swapping metadata date: %w", err)
+	}
+	return affectedOne(result)
+}
+
+func affectedOne(result sql.Result) (bool, error) {
+	n, err := result.RowsAffected()
+	if err != nil {
+		return false, fmt.Errorf("counting affected rows: %w", err)
+	}
+	return n == 1, nil
+}

internal/database/metadata_dates_test.go

@@ -0,0 +1,177 @@
+package database
+
+import (
+	"database/sql"
+	"strings"
+	"testing"
+	"time"
+)
+
+const testProxyURL = "http://proxy.example"
+
+func TestMetadataDateMissing(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "missing")
+		if err != nil || got != nil {
+			t.Errorf("GetMetadataDate() = %+v, %v; want nil, nil", got, err)
+		}
+	})
+}
+
+func TestMetadataDateInsertOnce(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		first := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)}
+		second := MetadataDate{ETag: `"b"`, LastModified: first.LastModified.Add(time.Second)}
+
+		inserted, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", first)
+		if err != nil || !inserted {
+			t.Fatalf("first InsertMetadataDate() = %v, %v; want true", inserted, err)
+		}
+		inserted, err = db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", second)
+		if err != nil || inserted {
+			t.Fatalf("second InsertMetadataDate() = %v, %v; want false, the row exists", inserted, err)
+		}
+
+		got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad")
+		if err != nil || got == nil || *got != first {
+			t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, first)
+		}
+	})
+}
+
+// A package's date is kept per proxy URL: each one rewrites the document to
+// different download URLs, so each serves a different representation.
+func TestMetadataDateKeyedByProxyURL(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)}
+		if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", date); err != nil {
+			t.Fatal(err)
+		}
+		inserted, err := db.InsertMetadataDate(testEcosystemNPM, "http://other.example", "left-pad", date)
+		if err != nil || !inserted {
+			t.Errorf("InsertMetadataDate() for another proxy URL = %v, %v; want true", inserted, err)
+		}
+	})
+}
+
+// SwapMetadataDate is a compare-and-swap: it only replaces the date it was
+// given, so of two replicas swapping from the same date only one succeeds.
+func TestMetadataDateSwap(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		a := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)}
+		b := MetadataDate{ETag: `"b"`, LastModified: a.LastModified.Add(time.Second)}
+		c := MetadataDate{ETag: `"c"`, LastModified: b.LastModified.Add(time.Second)}
+		if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a); err != nil {
+			t.Fatal(err)
+		}
+
+		swapped, err := db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, b)
+		if err != nil || !swapped {
+			t.Fatalf("SwapMetadataDate(a, b) = %v, %v; want true", swapped, err)
+		}
+		swapped, err = db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, c)
+		if err != nil || swapped {
+			t.Fatalf("SwapMetadataDate(a, c) after a was replaced = %v, %v; want false", swapped, err)
+		}
+
+		got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad")
+		if err != nil || got == nil || *got != b {
+			t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, b)
+		}
+	})
+}
+
+// Dates are whole seconds in UTC whatever the local zone, so every replica
+// reads back the date another one wrote.
+func TestMetadataDateIsUTCSeconds(t *testing.T) {
+	tokyo, err := time.LoadLocation("Asia/Tokyo")
+	if err != nil {
+		t.Skipf("time zone data unavailable: %v", err)
+	}
+	prev := time.Local
+	time.Local = tokyo
+	t.Cleanup(func() { time.Local = prev })
+
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		written := time.Date(2026, 10, 10, 21, 0, 1, 700_000_000, tokyo)
+		if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "tz", MetadataDate{ETag: `"a"`, LastModified: written}); err != nil {
+			t.Fatal(err)
+		}
+		got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "tz")
+		if err != nil || got == nil {
+			t.Fatalf("GetMetadataDate() = %+v, %v", got, err)
+		}
+		if want := time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC); !got.LastModified.Equal(want) || got.LastModified.Location() != time.UTC {
+			t.Errorf("LastModified read back as %v, want %v", got.LastModified, want)
+		}
+	})
+}
+
+// TestMetadataDatesMigrationKeepsCachedMetadata upgrades a database from
+// before migration 011 that holds cached npm and Composer metadata, and checks
+// that the cached rows come through unchanged and dates can be recorded.
+func TestMetadataDatesMigrationKeepsCachedMetadata(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		rollBackMetadataDates(t, db)
+		seeded := map[string]string{testEcosystemNPM: "left-pad", "composer": "vendor/pkg"}
+		before := make(map[string]*MetadataCacheEntry, len(seeded))
+		for ecosystem, name := range seeded {
+			before[ecosystem] = seedMetadataCache(t, db, ecosystem, name)
+		}
+
+		if err := db.MigrateSchema(); err != nil {
+			t.Fatalf("MigrateSchema() error = %v", err)
+		}
+
+		for ecosystem, name := range seeded {
+			after, err := db.GetMetadataCache(ecosystem, name)
+			if err != nil || after == nil || *after != *before[ecosystem] {
+				t.Errorf("metadata row %s/%s after migration = %+v, %v; want %+v", ecosystem, name, after, err, before[ecosystem])
+			}
+			date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)}
+			if inserted, err := db.InsertMetadataDate(ecosystem, testProxyURL, name, date); err != nil || !inserted {
+				t.Errorf("InsertMetadataDate(%s/%s) after migration = %v, %v; want true", ecosystem, name, inserted, err)
+			}
+		}
+	})
+}
+
+// seedMetadataCache stores a metadata row as the metadata cache writes one
+// and returns it as read back.
+func seedMetadataCache(t *testing.T, db *DB, ecosystem, name string) *MetadataCacheEntry {
+	t.Helper()
+	err := db.UpsertMetadataCache(&MetadataCacheEntry{
+		Ecosystem:     ecosystem,
+		Name:          name,
+		StoragePath:   "_metadata/" + ecosystem + "/" + name + "/metadata",
+		ETag:          sql.NullString{String: `"upstream"`, Valid: true},
+		ContentType:   sql.NullString{String: "application/json", Valid: true},
+		ContentDigest: sql.NullString{String: "sha256:" + strings.Repeat("ab", 32), Valid: true},
+		Size:          sql.NullInt64{Int64: 123, Valid: true},
+		LastModified:  sql.NullTime{Time: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), Valid: true},
+		FetchedAt:     sql.NullTime{Time: time.Now().UTC().Truncate(time.Second), Valid: true},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	entry, err := db.GetMetadataCache(ecosystem, name)
+	if err != nil || entry == nil {
+		t.Fatalf("GetMetadataCache(%s/%s) = %v, %v", ecosystem, name, entry, err)
+	}
+	return entry
+}
+
+// rollBackMetadataDates returns db to the schema from before migration 011,
+// which only added the metadata_dates table.
+func rollBackMetadataDates(t *testing.T, db *DB) {
+	t.Helper()
+	if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil {
+		t.Fatal(err)
+	}
+	if has, err := db.HasTable("metadata_dates"); err != nil || has {
+		t.Fatalf("HasTable(metadata_dates) after rollback = %v, %v; want false", has, err)
+	}
+}

internal/database/schema.go

@@ -119,6 +119,15 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name);
 
+CREATE TABLE IF NOT EXISTS metadata_dates (
+	ecosystem TEXT NOT NULL,
+	proxy_url TEXT NOT NULL,
+	name TEXT NOT NULL,
+	etag TEXT NOT NULL,
+	last_modified INTEGER NOT NULL,
+	PRIMARY KEY (ecosystem, proxy_url, name)
+);
+
 CREATE TABLE IF NOT EXISTS migrations (
 	name TEXT NOT NULL PRIMARY KEY,
 	applied_at DATETIME NOT NULL
@@ -228,6 +237,15 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name);
 
+CREATE TABLE IF NOT EXISTS metadata_dates (
+	ecosystem TEXT NOT NULL,
+	proxy_url TEXT NOT NULL,
+	name TEXT NOT NULL,
+	etag TEXT NOT NULL,
+	last_modified BIGINT NOT NULL,
+	PRIMARY KEY (ecosystem, proxy_url, name)
+);
+
 CREATE TABLE IF NOT EXISTS migrations (
 	name TEXT NOT NULL PRIMARY KEY,
 	applied_at TIMESTAMP NOT NULL
@@ -382,6 +400,7 @@ var migrations = []migration{
 	{"008_add_metadata_content_encoding", migrateAddMetadataContentEncoding},
 	{"009_add_pending_deletes", migrateAddPendingDeletes},
 	{"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex},
+	{"011_add_metadata_dates", migrateAddMetadataDates},
 }
 
 // isTableNotFound returns true if the error indicates a missing table.
@@ -681,6 +700,27 @@ func migrateAddPendingDeletesQueuedAtIndex(db *DB) error {
 	return nil
 }
 
+// migrateAddMetadataDates creates the table of Last-Modified dates for
+// rewritten metadata, which replicas sharing the database agree on.
+func migrateAddMetadataDates(db *DB) error {
+	seconds := "INTEGER"
+	if db.dialect == DialectPostgres {
+		seconds = "BIGINT"
+	}
+	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_dates (
+		ecosystem TEXT NOT NULL,
+		proxy_url TEXT NOT NULL,
+		name TEXT NOT NULL,
+		etag TEXT NOT NULL,
+		last_modified %s NOT NULL,
+		PRIMARY KEY (ecosystem, proxy_url, name)
+	)`, seconds)
+	if _, err := db.Exec(query); err != nil {
+		return fmt.Errorf("creating metadata_dates table: %w", err)
+	}
+	return nil
+}
+
 // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist.
 func (db *DB) EnsureMetadataCacheTable() error {
 	has, err := db.HasTable("metadata_cache")