diff --git a/harnesses/hyperliquid-frontends/cmd/script/agg.go b/harnesses/hyperliquid-frontends/cmd/script/agg.go index c90173691..bfa80b860 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/agg.go +++ b/harnesses/hyperliquid-frontends/cmd/script/agg.go @@ -372,6 +372,10 @@ func (a *Aggregator) publish() { } } + // day key -> the latest fill second seen across every builder. Used to + // tell a short day from a truncated feed; see publishCoverage. + cohortLastFill := map[string]int64{} + rows := make(map[string]builderWindow, len(a.builders)) var cohortVol float64 for _, b := range a.builders { @@ -393,6 +397,12 @@ func (a *Aggregator) publish() { continue } s := a.merged(b, d) + if s != nil && s.lastFillS > cohortLastFill[dayKey(d)] { + // Furthest fill any builder saw that day. One builder going + // quiet in the evening proves nothing; the whole cohort going + // quiet at the same minute is the feed being cut short. + cohortLastFill[dayKey(d)] = s.lastFillS + } if i == 0 { w.day = s w.dayKnown = s != nil || a.allAbsent(b, d) @@ -483,6 +493,7 @@ func (a *Aggregator) publish() { hlDataDay.Set(float64(D.Unix())) hlDataDayEnd.Set(float64(D.Unix() + daySec)) + a.publishCoverage(D, windowStart, cohortLastFill) // Liveness means "the feed was reachable and the gauges were rebuilt", // not just that the process is up: a pass with transport failures // leaves the tick alone so the success query goes false after 2 h. @@ -829,3 +840,56 @@ func (a *Aggregator) writeLedgerDay(b Builder, d time.Time) { a.state.setLedger(b.Slug, dayKey(d), dayTotals{}) } } + +// dayCoverageHours is how far into a UTC day the cohort's furthest fill +// reached. A complete day lands within minutes of 24; anything materially +// short means the published file stops before the day does. +func dayCoverageHours(day time.Time, lastFillS int64) float64 { + if lastFillS <= 0 { + return 0 + } + h := float64(lastFillS-day.Unix()) / 3600 + switch { + case h < 0: + return 0 + case h > 24: + return 24 + } + return h +} + +// coverageCompleteHours is the point past which a day counts as whole. +// +// Hyperliquid's per-builder export started stopping at roughly 12:11 UTC on +// 2026-09-22 and has every day since: 21 September ran to 23:59:54, 22 +// September to 12:27:11, and 2 October to 12:10:58, with the file stable +// across re-downloads, so this is the published artefact and not a partial +// fetch. Summing such a day beside a whole one understates it by about half +// and says nothing, which is the shape of error this harness exists to +// avoid. 23 leaves an hour of slack for a genuinely quiet late evening +// across the entire cohort, which has never happened in the mirror. +const coverageCompleteHours = 23 + +// publishCoverage reports how much of each day the feed actually carried. +// The figures stay published either way: the fix for a truncated upstream is +// to say so, not to drop the day and let the window silently shorten. +func (a *Aggregator) publishCoverage(D, windowStart time.Time, lastFill map[string]int64) { + cov := dayCoverageHours(D, lastFill[dayKey(D)]) + hlDayCoverageHours.Set(cov) + if cov < coverageCompleteHours { + hlDayTruncated.Set(1) + } else { + hlDayTruncated.Set(0) + } + truncated := 0 + for d := windowStart; !d.After(D); d = d.AddDate(0, 0, 1) { + ls, ok := lastFill[dayKey(d)] + if !ok { + continue // no file for that day: already counted elsewhere + } + if dayCoverageHours(d, ls) < coverageCompleteHours { + truncated++ + } + } + hlTruncatedDaysWindow.Set(float64(truncated)) +} diff --git a/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go b/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go new file mode 100644 index 000000000..9bdc987e3 --- /dev/null +++ b/harnesses/hyperliquid-frontends/cmd/script/coverage_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "math" + "testing" + "time" +) + +func utc(y int, m time.Month, d int) time.Time { + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +// at returns the unix second of hh:mm:ss inside day. +func at(day time.Time, hh, mm, ss int) int64 { + return day.Add(time.Duration(hh)*time.Hour + time.Duration(mm)*time.Minute + time.Duration(ss)*time.Second).Unix() +} + +// The numbers are the real ones, read from Hyperliquid's published export +// for Trust Wallet (0x5af1…5f71) on 2026-10-04. They are the reason this +// check exists, so they are what it is tested against. +func TestDayCoverageOnTheRealFeed(t *testing.T) { + cases := []struct { + day time.Time + hh, mm, ss int + wantHours float64 + truncated bool + note string + }{ + {utc(2026, 9, 10), 23, 59, 57, 24.0, false, "whole day, the shape before the break"}, + {utc(2026, 9, 21), 23, 59, 54, 24.0, false, "last whole day published"}, + {utc(2026, 9, 22), 12, 27, 11, 12.45, true, "first truncated day"}, + {utc(2026, 9, 25), 12, 13, 31, 12.23, true, ""}, + {utc(2026, 9, 30), 12, 11, 42, 12.20, true, ""}, + {utc(2026, 10, 2), 12, 10, 58, 12.18, true, "the day the discrepancy was reported on"}, + } + for _, c := range cases { + got := dayCoverageHours(c.day, at(c.day, c.hh, c.mm, c.ss)) + if math.Abs(got-c.wantHours) > 0.01 { + t.Errorf("%s: coverage = %.2f h, want %.2f (%s)", c.day.Format("2006-01-02"), got, c.wantHours, c.note) + } + if tr := got < coverageCompleteHours; tr != c.truncated { + t.Errorf("%s: truncated = %v at %.2f h, want %v (%s)", c.day.Format("2006-01-02"), tr, got, c.truncated, c.note) + } + } +} + +func TestCoverageEdges(t *testing.T) { + d := utc(2026, 10, 2) + if got := dayCoverageHours(d, 0); got != 0 { + t.Errorf("no fill at all = %v, want 0", got) + } + // A day with no file must not read as a complete one. + if 0 >= coverageCompleteHours { + t.Error("zero coverage must count as truncated") + } + // Clock skew or a stray row must not invent coverage. + if got := dayCoverageHours(d, d.Unix()-3600); got != 0 { + t.Errorf("a fill before the day started = %v, want 0", got) + } + if got := dayCoverageHours(d, d.Unix()+30*3600); got != 24 { + t.Errorf("a fill past the day's end = %v, want it clamped to 24", got) + } +} + +// The threshold has to let a genuinely quiet late evening through while +// still catching the real break. 23 h sits between them with room on both +// sides: the widest truncation seen is 12.45 h and a whole day is 24. +func TestThresholdSeparatesQuietFromTruncated(t *testing.T) { + d := utc(2026, 10, 2) + if dayCoverageHours(d, at(d, 23, 10, 0)) < coverageCompleteHours { + t.Error("a cohort whose last fill is 23:10 is quiet, not truncated") + } + if dayCoverageHours(d, at(d, 12, 27, 11)) >= coverageCompleteHours { + t.Error("12:27 is the real truncation and must be caught") + } +} diff --git a/harnesses/hyperliquid-frontends/cmd/script/metrics.go b/harnesses/hyperliquid-frontends/cmd/script/metrics.go index 302bd71ec..a7a1cc9b5 100644 --- a/harnesses/hyperliquid-frontends/cmd/script/metrics.go +++ b/harnesses/hyperliquid-frontends/cmd/script/metrics.go @@ -37,6 +37,19 @@ var ( hlPercentile = gauge("hl_frontend_volume_by_percentile_30d_v2", "Share (0..1) of the builder's 30-day notional contributed by wallets in the rank bucket: top1, p1_5, p5_10, p10_25, p25_50, rest.", "builder", "bucket") hlCoinShare = gauge("hl_frontend_coin_volume_share_24h_v2", "Share (0..1) of the feed day's notional per coin, top 15 coins plus 'other'.", "builder", "coin") + hlDayCoverageHours = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_day_coverage_hours_v2", + Help: "How far into the headline feed day the cohort's furthest fill reached, in hours. A whole day lands within minutes of 24. Hyperliquid's per-builder export has stopped at roughly 12:11 UTC every day since 2026-09-22, so a reading near 12 means the published file ends before the day does and the 24h figures describe half a day.", + }) + hlDayTruncated = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_day_truncated_v2", + Help: "1 when the headline feed day covers less than 23 hours. The figures are still published: the honest answer to a truncated upstream is to say so, not to drop the day and let the window silently shorten.", + }) + hlTruncatedDaysWindow = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hl_frontend_truncated_days_window_v2", + Help: "Days inside the 30-day window whose published file stops short of 23 hours. Every windowed total is understated by roughly half a day for each one counted here.", + }) + hlDataDay = promauto.NewGauge(prometheus.GaugeOpts{ Name: "hl_frontend_data_day_unix_v2", Help: "UTC midnight (unix seconds) that opens the feed day the 24h gauges describe.",