From bfd45b13c097a1c953c257185fec33cfcb065292 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:12:40 +0200 Subject: [PATCH] Hyperliquid has published half a day since 22 September and we summed it as a whole one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported as "our stats don't match CoinMarketMan, sometimes +50% diff". They do not, and the cause is upstream. Hyperliquid's per-builder fills export stops partway through each day. Trust Wallet (0x5af1…5f71), last fill in the published file: 10 Sep 23:59:57 whole 21 Sep 23:59:54 whole 22 Sep 12:27:11 truncated 25 Sep 12:13:31 truncated 30 Sep 12:11:42 truncated 2 Oct 12:10:58 truncated Every day since 22 September ends within three minutes of 12:11 UTC. The file is stable across re-downloads (identical md5), so this is the published artefact and not a partial fetch. Checked against the live info API for one user on 2 October: the CSV holds 953 fills, the API 1,299 carrying a builder fee, and every CSV fill is present in the API with none the other way. The export is a strict subset. Both sides imply the same 0.0950% fee rate, which is Trust Wallet's, so the extra fills are its own and not another builder's. By hour: 04-11 kept 100%, hour 12 kept 47%, hours 13-15 kept nothing. So our figures were a faithful read of an upstream that had quietly halved. Fees and volume were understated together, the rates stayed right, and nothing in the pipeline could tell. This publishes the coverage instead of hiding it. The cohort's furthest fill per day is already collected during the pass that builds the windows, so the check costs no extra parsing: one builder going quiet in the evening proves nothing, the whole cohort going quiet at the same minute is the feed being cut short. hl_frontend_day_coverage_hours_v2 hours the headline day reached hl_frontend_day_truncated_v2 1 when it is under 23 hl_frontend_truncated_days_window_v2 such days inside the 30-day window Figures stay published. The answer to a truncated upstream is to say so, not to drop the day and let the window silently shorten. The node is not an alternative: it is running on SGP but has not advanced since 3 October 09:10, looping on "could not read abci state: early eof", with 10 of 24 hours for that day and nothing for the 4th. --- .../hyperliquid-frontends/cmd/script/agg.go | 64 ++++++++++++++++ .../cmd/script/coverage_test.go | 76 +++++++++++++++++++ .../cmd/script/metrics.go | 13 ++++ 3 files changed, 153 insertions(+) create mode 100644 harnesses/hyperliquid-frontends/cmd/script/coverage_test.go diff --git a/harnesses/hyperliquid-frontends/cmd/script/agg.go b/harnesses/hyperliquid-frontends/cmd/script/agg.go index c90173691..bfa80b860 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/agg.go +++ b/harnesses/hyperliquid-frontends/cmd/script/agg.go @@ -372,6 +372,10 @@ func (a *Aggregator) publish() { } } + // day key -> the latest fill second seen across every builder. Used to + // tell a short day from a truncated feed; see publishCoverage. + cohortLastFill := map[string]int64{} + rows := make(map[string]builderWindow, len(a.builders)) var cohortVol float64 for _, b := range a.builders { @@ -393,6 +397,12 @@ func (a *Aggregator) publish() { continue } s := a.merged(b, d) + if s != nil && s.lastFillS > cohortLastFill[dayKey(d)] { + // Furthest fill any builder saw that day. One builder going + // quiet in the evening proves nothing; the whole cohort going + // quiet at the same minute is the feed being cut short. + cohortLastFill[dayKey(d)] = s.lastFillS + } if i == 0 { w.day = s w.dayKnown = s != nil || a.allAbsent(b, d) @@ -483,6 +493,7 @@ func (a *Aggregator) publish() { hlDataDay.Set(float64(D.Unix())) hlDataDayEnd.Set(float64(D.Unix() + daySec)) + a.publishCoverage(D, windowStart, cohortLastFill) // Liveness means "the feed was reachable and the gauges were rebuilt", // not just that the process is up: a pass with transport failures // leaves the tick alone so the success query goes false after 2 h. @@ -829,3 +840,56 @@ func (a *Aggregator) writeLedgerDay(b Builder, d time.Time) { a.state.setLedger(b.Slug, dayKey(d), dayTotals{}) } } + +// dayCoverageHours is how far into a UTC day the cohort's furthest fill +// reached. A complete day lands within minutes of 24; anything materially +// short means the published file stops before the day does. +func dayCoverageHours(day time.Time, lastFillS int64) float64 { + if lastFillS <= 0 { + return 0 + } + h := float64(lastFillS-day.Unix()) / 3600 + switch { + case h < 0: + return 0 + case h > 24: + return 24 + } + return h +} + +// coverageCompleteHours is the point past which a day counts as whole. +// +// Hyperliquid's per-builder export started stopping at roughly 12:11 UTC on +// 2026-09-22 and has every day since: 21 September ran to 23:59:54, 22 +// September to 12:27:11, and 2 October to 12:10:58, with the file stable +// across re-downloads, so this is the published artefact and not a partial +// fetch. Summing such a day beside a whole one understates it by about half +// and says nothing, which is the shape of error this harness exists to +// avoid. 23 leaves an hour of slack for a genuinely quiet late evening +// across the entire cohort, which has never happened in the mirror. +const coverageCompleteHours = 23 + +// publishCoverage reports how much of each day the feed actually carried. +// The figures stay published either way: the fix for a truncated upstream is +// to say so, not to drop the day and let the window silently shorten. +func (a *Aggregator) publishCoverage(D, windowStart time.Time, lastFill map[string]int64) { + cov := dayCoverageHours(D, lastFill[dayKey(D)]) + hlDayCoverageHours.Set(cov) + if cov < coverageCompleteHours { + hlDayTruncated.Set(1) + } else { + hlDayTruncated.Set(0) + } + truncated := 0 + for d := windowStart; !d.After(D); d = d.AddDate(0, 0, 1) { + ls, ok := lastFill[dayKey(d)] + if !ok { + continue // no file for that day: already counted elsewhere + } + if dayCoverageHours(d, ls) < coverageCompleteHours { + truncated++ + } + } + hlTruncatedDaysWindow.Set(float64(truncated)) +} diff --git a/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go b/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go new file mode 100644 index 000000000..9bdc987e3 --- /dev/null +++ b/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "math" + "testing" + "time" +) + +func utc(y int, m time.Month, d int) time.Time { + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +// at returns the unix second of hh:mm:ss inside day. +func at(day time.Time, hh, mm, ss int) int64 { + return day.Add(time.Duration(hh)*time.Hour + time.Duration(mm)*time.Minute + time.Duration(ss)*time.Second).Unix() +} + +// The numbers are the real ones, read from Hyperliquid's published export +// for Trust Wallet (0x5af1…5f71) on 2026-10-04. They are the reason this +// check exists, so they are what it is tested against. +func TestDayCoverageOnTheRealFeed(t *testing.T) { + cases := []struct { + day time.Time + hh, mm, ss int + wantHours float64 + truncated bool + note string + }{ + {utc(2026, 9, 10), 23, 59, 57, 24.0, false, "whole day, the shape before the break"}, + {utc(2026, 9, 21), 23, 59, 54, 24.0, false, "last whole day published"}, + {utc(2026, 9, 22), 12, 27, 11, 12.45, true, "first truncated day"}, + {utc(2026, 9, 25), 12, 13, 31, 12.23, true, ""}, + {utc(2026, 9, 30), 12, 11, 42, 12.20, true, ""}, + {utc(2026, 10, 2), 12, 10, 58, 12.18, true, "the day the discrepancy was reported on"}, + } + for _, c := range cases { + got := dayCoverageHours(c.day, at(c.day, c.hh, c.mm, c.ss)) + if math.Abs(got-c.wantHours) > 0.01 { + t.Errorf("%s: coverage = %.2f h, want %.2f (%s)", c.day.Format("2006-01-02"), got, c.wantHours, c.note) + } + if tr := got < coverageCompleteHours; tr != c.truncated { + t.Errorf("%s: truncated = %v at %.2f h, want %v (%s)", c.day.Format("2006-01-02"), tr, got, c.truncated, c.note) + } + } +} + +func TestCoverageEdges(t *testing.T) { + d := utc(2026, 10, 2) + if got := dayCoverageHours(d, 0); got != 0 { + t.Errorf("no fill at all = %v, want 0", got) + } + // A day with no file must not read as a complete one. + if 0 >= coverageCompleteHours { + t.Error("zero coverage must count as truncated") + } + // Clock skew or a stray row must not invent coverage. + if got := dayCoverageHours(d, d.Unix()-3600); got != 0 { + t.Errorf("a fill before the day started = %v, want 0", got) + } + if got := dayCoverageHours(d, d.Unix()+30*3600); got != 24 { + t.Errorf("a fill past the day's end = %v, want it clamped to 24", got) + } +} + +// The threshold has to let a genuinely quiet late evening through while +// still catching the real break. 23 h sits between them with room on both +// sides: the widest truncation seen is 12.45 h and a whole day is 24. +func TestThresholdSeparatesQuietFromTruncated(t *testing.T) { + d := utc(2026, 10, 2) + if dayCoverageHours(d, at(d, 23, 10, 0)) < coverageCompleteHours { + t.Error("a cohort whose last fill is 23:10 is quiet, not truncated") + } + if dayCoverageHours(d, at(d, 12, 27, 11)) >= coverageCompleteHours { + t.Error("12:27 is the real truncation and must be caught") + } +} diff --git a/harnesses/hyperliquid-frontends/cmd/script/metrics.go b/harnesses/hyperliquid-frontends/cmd/script/metrics.go index 302bd71ec..a7a1cc9b5 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/metrics.go +++ b/harnesses/hyperliquid-frontends/cmd/script/metrics.go @@ -37,6 +37,19 @@ var ( hlPercentile = gauge("hl_frontend_volume_by_percentile_30d_v2", "Share (0..1) of the builder's 30-day notional contributed by wallets in the rank bucket: top1, p1_5, p5_10, p10_25, p25_50, rest.", "builder", "bucket") hlCoinShare = gauge("hl_frontend_coin_volume_share_24h_v2", "Share (0..1) of the feed day's notional per coin, top 15 coins plus 'other'.", "builder", "coin") + hlDayCoverageHours = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_day_coverage_hours_v2", + Help: "How far into the headline feed day the cohort's furthest fill reached, in hours. A whole day lands within minutes of 24. Hyperliquid's per-builder export has stopped at roughly 12:11 UTC every day since 2026-09-22, so a reading near 12 means the published file ends before the day does and the 24h figures describe half a day.", + }) + hlDayTruncated = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_day_truncated_v2", + Help: "1 when the headline feed day covers less than 23 hours. The figures are still published: the honest answer to a truncated upstream is to say so, not to drop the day and let the window silently shorten.", + }) + hlTruncatedDaysWindow = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_truncated_days_window_v2", + Help: "Days inside the 30-day window whose published file stops short of 23 hours. Every windowed total is understated by roughly half a day for each one counted here.", + }) + hlDataDay = promauto.NewGauge(prometheus.GaugeOpts{ Name: "hl_frontend_data_day_unix_v2", Help: "UTC midnight (unix seconds) that opens the feed day the 24h gauges describe.",