From 0e2859c862fa75660096802f1889b51e0291baf6 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:13:41 +0200 Subject: [PATCH] the mirror took the first copy of a day and never looked again Hyperliquid publishes a day short and sometimes completes it later. 3 October was still a 403 at 22:00 on the 4th and was whole, to 23:59:23, by 01:00 on the 5th. We only got the complete one by luck: the file did not exist at the usual fetch hour, so our first successful download landed after the upstream had finished writing it. 4 October shows the other outcome. It was fetched at 00:59 while still truncated at 12:13:24, and nothing in the harness would ever have gone back for it. syncWindow skipped any day already on disk. That is right for a day that cannot change and wrong for one that can, and the harness had no way to tell the two apart. Recent days are now revalidated instead of skipped. The conditional token is the mirrored file's own mtime, which already carries the CDN's Last-Modified (fetch has stamped it for the settle check since the feed edition), so no new state is stored. Measured on the live CDN: ETag and Last-Modified are both served and If-Modified-Since answers 304, which is what makes re-checking every recent day on every pass affordable. Only days inside the grace window are re-checked. Past it the upstream has never once gone back and filled a day in: 22 September to 2 October all stop near 12:11 UTC and have not moved in the two weeks since, byte for byte identical across re-downloads. Re-checking those would be pure load. A 304 is not a failure and does not count toward the sync failure budget. A 200 on a day we already held means the upstream republished, so the cached parse is of the old half and is dropped. This recovers 4 October if Hyperliquid completes it, and every day after. It does not recover 22 September to 2 October, which are truncated at the source for good; those need the API backfill or they age out of the window in November. --- .../hyperliquid-frontends/cmd/script/agg.go | 38 ++++++- .../hyperliquid-frontends/cmd/script/feed.go | 21 +++- .../cmd/script/refetch_test.go | 102 ++++++++++++++++++ 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 harnesses/hyperliquid-frontends/cmd/script/refetch_test.go diff --git a/harnesses/hyperliquid-frontends/cmd/script/agg.go b/harnesses/hyperliquid-frontends/cmd/script/agg.go index bfa80b860..e026ad48d 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/agg.go +++ b/harnesses/hyperliquid-frontends/cmd/script/agg.go @@ -131,8 +131,16 @@ func (a *Aggregator) runJobs(ctx context.Context, jobs []fetchJob) int { if ctx.Err() != nil { return } + had := a.mirror.present(j.addr, j.day) err := a.mirror.fetch(ctx, j.addr, j.day) - if err != nil && err != errFeedAbsent { + if err == nil && had { + // Revalidation returned a new body: the CDN completed a + // day it had published short. The parse we cached is of + // the old half, so drop it. + a.forgetDay(j.addr, j.day) + log.Printf("feed %s %s: upstream republished, cached parse dropped", j.addr, j.day.Format(dayFormat)) + } + if err != nil && err != errFeedAbsent && err != errFeedUnchanged { fmu.Lock() failures++ fmu.Unlock() @@ -166,7 +174,21 @@ func (a *Aggregator) syncWindow(ctx context.Context) { recent := d.After(to.AddDate(0, 0, -a.graceDays)) for _, addr := range a.addrs { if a.mirror.present(addr, d) { - continue + // Not "we have it, we are done". Hyperliquid publishes a day + // short and completes it later: 3 October was still a 403 at + // 22:00 on the 4th and was whole by 01:00 on the 5th, while + // 22 September to 2 October stopped near 12:11 UTC and have + // never moved since. Holding the first copy for ever is how a + // half day becomes permanent on our side. + // + // Revalidation is conditional (If-Modified-Since from the + // file's own mtime), so an unchanged day costs a 304 and + // nothing else. Only days still inside the grace window are + // re-checked: past it the upstream has never once gone back + // and filled a day in, so it would be pure load. + if !recent { + continue + } } // A 403 on an old day is final (no fills). On a recent day every // missing file is re-requested on every pass, so by the time the @@ -893,3 +915,15 @@ func (a *Aggregator) publishCoverage(D, windowStart time.Time, lastFill map[stri } hlTruncatedDaysWindow.Set(float64(truncated)) } + +// forgetDay drops one cached parse so the next read re-reads the file from +// disk. Used when a revalidation replaces a mirrored day with a longer one. +func (a *Aggregator) forgetDay(addr string, day time.Time) { + a.mu.Lock() + defer a.mu.Unlock() + // Keyed exactly as summary() keys it: the address as the caller passes + // it, which is the form a.addrs carries. + if m, ok := a.cache[addr]; ok { + delete(m, dayKey(day)) + } +} diff --git a/harnesses/hyperliquid-frontends/cmd/script/feed.go b/harnesses/hyperliquid-frontends/cmd/script/feed.go index f944d1dce..361d09da1 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/feed.go +++ b/harnesses/hyperliquid-frontends/cmd/script/feed.go @@ -23,7 +23,9 @@ import ( // 02:45 to 03:30 UTC). A missing file answers 403, which is also what a // builder with no fills that day gets, so "not published yet" and "no // fills" are told apart at cohort level (see availability.go). -const feedURLTemplate = "https://stats-data.hyperliquid.xyz/Mainnet/builder_fills/%s/%s.csv.lz4" +// feedURLTemplate is a var, not a const, so the revalidation path can be +// exercised against a local server in tests. Never reassigned in production. +var feedURLTemplate = "https://stats-data.hyperliquid.xyz/Mainnet/builder_fills/%s/%s.csv.lz4" const dayFormat = "20060102" @@ -31,6 +33,10 @@ var feedHTTP = &http.Client{Timeout: 120 * time.Second} var errFeedAbsent = errors.New("feed: no file for that builder and day") +// errFeedUnchanged is the CDN answering 304 to a revalidation: the mirrored +// copy is still the newest one published. Not a failure. +var errFeedUnchanged = errors.New("feed: not modified") + // Mirror keeps a local copy of every fetched CSV under // /builder_fills/
/.csv.lz4, plus a // .absent marker (containing the unix time of the check) when the @@ -129,6 +135,15 @@ func (m *Mirror) fetch(ctx context.Context, addr string, day time.Time) error { return err } req.Header.Set("User-Agent", "openchainbench-hl-frontends/2 (+https://openchainbench.com)") + // Revalidate rather than re-download. The mirrored file already + // carries the CDN's own Last-Modified as its mtime (stamped below), + // so it is the conditional token and no extra state is needed. The + // CDN answers 304, which is what makes it affordable to re-check + // every recent day on every pass instead of trusting the first copy + // for ever. + if st, err := os.Stat(m.filePath(addr, day)); err == nil && st.Size() > 0 { + req.Header.Set("If-Modified-Since", st.ModTime().UTC().Format(http.TimeFormat)) + } resp, err := feedHTTP.Do(req) if err != nil { lastErr = err @@ -136,6 +151,10 @@ func (m *Mirror) fetch(ctx context.Context, addr string, day time.Time) error { continue } switch { + case resp.StatusCode == http.StatusNotModified: + resp.Body.Close() + feedFetchTotal.WithLabelValues("unchanged").Inc() + return errFeedUnchanged case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound: resp.Body.Close() feedFetchTotal.WithLabelValues("absent").Inc() diff --git a/harnesses/hyperliquid-frontends/cmd/script/refetch_test.go b/harnesses/hyperliquid-frontends/cmd/script/refetch_test.go new file mode 100644 index 000000000..6aaa36f00 --- /dev/null +++ b/harnesses/hyperliquid-frontends/cmd/script/refetch_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" +) + +// The behaviour under test is the one that let a half day become permanent: +// the mirror used to take the first copy of a day and never look again, so +// when Hyperliquid published 3 October short and completed it two days +// later, our side would have kept the short one for ever. +func TestRevalidationPicksUpACompletedDay(t *testing.T) { + short := []byte("short-day-body") + long := []byte("the-same-day-once-upstream-finished-writing-it") + + body := short + modified := time.Date(2026, 10, 5, 0, 59, 25, 0, time.UTC) + var conditional, served int + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if ims := r.Header.Get("If-Modified-Since"); ims != "" { + conditional++ + if since, err := http.ParseTime(ims); err == nil && !modified.After(since) { + w.WriteHeader(http.StatusNotModified) + return + } + } + served++ + w.Header().Set("Last-Modified", modified.UTC().Format(http.TimeFormat)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + })) + defer srv.Close() + + old := feedURLTemplate + feedURLTemplate = srv.URL + "/%s/%s.csv.lz4" + defer func() { feedURLTemplate = old }() + + m, err := newMirror(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + addr, day := "0xabc", time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC) + + // First pass: nothing on disk, so no condition is sent and we take the + // short body the upstream has so far. + if err := m.fetch(ctx, addr, day); err != nil { + t.Fatalf("first fetch: %v", err) + } + if conditional != 0 { + t.Errorf("first fetch sent a condition with nothing on disk") + } + if got, _ := os.ReadFile(m.filePath(addr, day)); string(got) != string(short) { + t.Fatalf("first fetch stored %q", got) + } + + // Second pass: unchanged upstream. This must cost a 304 and leave the + // file alone, which is what makes re-checking every recent day cheap + // enough to do on every sync. + if err := m.fetch(ctx, addr, day); err != errFeedUnchanged { + t.Fatalf("revalidation of an unchanged day = %v, want errFeedUnchanged", err) + } + if conditional != 1 || served != 1 { + t.Errorf("conditional=%d served=%d, want 1 and 1: the body was re-sent", conditional, served) + } + + // Upstream completes the day. + body = long + modified = modified.Add(2 * time.Hour) + if err := m.fetch(ctx, addr, day); err != nil { + t.Fatalf("fetch after upstream republished: %v", err) + } + got, _ := os.ReadFile(m.filePath(addr, day)) + if string(got) != string(long) { + t.Fatalf("after the upstream completed the day the mirror still holds %q", got) + } +} + +// A day the upstream never had must still read as absent, not as unchanged: +// the two are different answers and the caller treats them differently. +func TestAbsentDayIsNotUnchanged(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + old := feedURLTemplate + feedURLTemplate = srv.URL + "/%s/%s.csv.lz4" + defer func() { feedURLTemplate = old }() + + m, err := newMirror(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := m.fetch(context.Background(), "0xabc", time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC)); err != errFeedAbsent { + t.Fatalf("403 = %v, want errFeedAbsent", err) + } +}