Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions harnesses/hyperliquid-frontends/cmd/script/agg.go
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,10 @@ func (a *Aggregator) publish() {
}
}

// day key -> the latest fill second seen across every builder. Used to
// tell a short day from a truncated feed; see publishCoverage.
cohortLastFill := map[string]int64{}

rows := make(map[string]builderWindow, len(a.builders))
var cohortVol float64
for _, b := range a.builders {
Expand All @@ -393,6 +397,12 @@ func (a *Aggregator) publish() {
continue
}
s := a.merged(b, d)
if s != nil && s.lastFillS > cohortLastFill[dayKey(d)] {
// Furthest fill any builder saw that day. One builder going
// quiet in the evening proves nothing; the whole cohort going
// quiet at the same minute is the feed being cut short.
cohortLastFill[dayKey(d)] = s.lastFillS
}
if i == 0 {
w.day = s
w.dayKnown = s != nil || a.allAbsent(b, d)
Expand Down Expand Up @@ -483,6 +493,7 @@ func (a *Aggregator) publish() {

hlDataDay.Set(float64(D.Unix()))
hlDataDayEnd.Set(float64(D.Unix() + daySec))
a.publishCoverage(D, windowStart, cohortLastFill)
// Liveness means "the feed was reachable and the gauges were rebuilt",
// not just that the process is up: a pass with transport failures
// leaves the tick alone so the success query goes false after 2 h.
Expand Down Expand Up @@ -829,3 +840,56 @@ func (a *Aggregator) writeLedgerDay(b Builder, d time.Time) {
a.state.setLedger(b.Slug, dayKey(d), dayTotals{})
}
}

// dayCoverageHours is how far into a UTC day the cohort's furthest fill
// reached. A complete day lands within minutes of 24; anything materially
// short means the published file stops before the day does.
func dayCoverageHours(day time.Time, lastFillS int64) float64 {
if lastFillS <= 0 {
return 0
}
h := float64(lastFillS-day.Unix()) / 3600
switch {
case h < 0:
return 0
case h > 24:
return 24
}
return h
}

// coverageCompleteHours is the point past which a day counts as whole.
//
// Hyperliquid's per-builder export started stopping at roughly 12:11 UTC on
// 2026-09-22 and has every day since: 21 September ran to 23:59:54, 22
// September to 12:27:11, and 2 October to 12:10:58, with the file stable
// across re-downloads, so this is the published artefact and not a partial
// fetch. Summing such a day beside a whole one understates it by about half
// and says nothing, which is the shape of error this harness exists to
// avoid. 23 leaves an hour of slack for a genuinely quiet late evening
// across the entire cohort, which has never happened in the mirror.
const coverageCompleteHours = 23

// publishCoverage reports how much of each day the feed actually carried.
// The figures stay published either way: the fix for a truncated upstream is
// to say so, not to drop the day and let the window silently shorten.
func (a *Aggregator) publishCoverage(D, windowStart time.Time, lastFill map[string]int64) {
cov := dayCoverageHours(D, lastFill[dayKey(D)])
hlDayCoverageHours.Set(cov)
if cov < coverageCompleteHours {
hlDayTruncated.Set(1)
} else {
hlDayTruncated.Set(0)
}
truncated := 0
for d := windowStart; !d.After(D); d = d.AddDate(0, 0, 1) {
ls, ok := lastFill[dayKey(d)]
if !ok {
continue // no file for that day: already counted elsewhere
}
if dayCoverageHours(d, ls) < coverageCompleteHours {
truncated++
}
}
hlTruncatedDaysWindow.Set(float64(truncated))
}
76 changes: 76 additions & 0 deletions harnesses/hyperliquid-frontends/cmd/script/coverage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package main

import (
"math"
"testing"
"time"
)

func utc(y int, m time.Month, d int) time.Time {
return time.Date(y, m, d, 0, 0, 0, 0, time.UTC)
}

// at returns the unix second of hh:mm:ss inside day.
func at(day time.Time, hh, mm, ss int) int64 {
return day.Add(time.Duration(hh)*time.Hour + time.Duration(mm)*time.Minute + time.Duration(ss)*time.Second).Unix()
}

// The numbers are the real ones, read from Hyperliquid's published export
// for Trust Wallet (0x5af1…5f71) on 2026-10-04. They are the reason this
// check exists, so they are what it is tested against.
func TestDayCoverageOnTheRealFeed(t *testing.T) {
cases := []struct {
day time.Time
hh, mm, ss int
wantHours float64
truncated bool
note string
}{
{utc(2026, 9, 10), 23, 59, 57, 24.0, false, "whole day, the shape before the break"},
{utc(2026, 9, 21), 23, 59, 54, 24.0, false, "last whole day published"},
{utc(2026, 9, 22), 12, 27, 11, 12.45, true, "first truncated day"},
{utc(2026, 9, 25), 12, 13, 31, 12.23, true, ""},
{utc(2026, 9, 30), 12, 11, 42, 12.20, true, ""},
{utc(2026, 10, 2), 12, 10, 58, 12.18, true, "the day the discrepancy was reported on"},
}
for _, c := range cases {
got := dayCoverageHours(c.day, at(c.day, c.hh, c.mm, c.ss))
if math.Abs(got-c.wantHours) > 0.01 {
t.Errorf("%s: coverage = %.2f h, want %.2f (%s)", c.day.Format("2006-01-02"), got, c.wantHours, c.note)
}
if tr := got < coverageCompleteHours; tr != c.truncated {
t.Errorf("%s: truncated = %v at %.2f h, want %v (%s)", c.day.Format("2006-01-02"), tr, got, c.truncated, c.note)
}
}
}

func TestCoverageEdges(t *testing.T) {
d := utc(2026, 10, 2)
if got := dayCoverageHours(d, 0); got != 0 {
t.Errorf("no fill at all = %v, want 0", got)
}
// A day with no file must not read as a complete one.
if 0 >= coverageCompleteHours {
t.Error("zero coverage must count as truncated")
}
// Clock skew or a stray row must not invent coverage.
if got := dayCoverageHours(d, d.Unix()-3600); got != 0 {
t.Errorf("a fill before the day started = %v, want 0", got)
}
if got := dayCoverageHours(d, d.Unix()+30*3600); got != 24 {
t.Errorf("a fill past the day's end = %v, want it clamped to 24", got)
}
}

// The threshold has to let a genuinely quiet late evening through while
// still catching the real break. 23 h sits between them with room on both
// sides: the widest truncation seen is 12.45 h and a whole day is 24.
func TestThresholdSeparatesQuietFromTruncated(t *testing.T) {
d := utc(2026, 10, 2)
if dayCoverageHours(d, at(d, 23, 10, 0)) < coverageCompleteHours {
t.Error("a cohort whose last fill is 23:10 is quiet, not truncated")
}
if dayCoverageHours(d, at(d, 12, 27, 11)) >= coverageCompleteHours {
t.Error("12:27 is the real truncation and must be caught")
}
}
13 changes: 13 additions & 0 deletions harnesses/hyperliquid-frontends/cmd/script/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,19 @@ var (
hlPercentile = gauge("hl_frontend_volume_by_percentile_30d_v2", "Share (0..1) of the builder's 30-day notional contributed by wallets in the rank bucket: top1, p1_5, p5_10, p10_25, p25_50, rest.", "builder", "bucket")
hlCoinShare = gauge("hl_frontend_coin_volume_share_24h_v2", "Share (0..1) of the feed day's notional per coin, top 15 coins plus 'other'.", "builder", "coin")

hlDayCoverageHours = promauto.NewGauge(prometheus.GaugeOpts{
Name: "hl_frontend_day_coverage_hours_v2",
Help: "How far into the headline feed day the cohort's furthest fill reached, in hours. A whole day lands within minutes of 24. Hyperliquid's per-builder export has stopped at roughly 12:11 UTC every day since 2026-09-22, so a reading near 12 means the published file ends before the day does and the 24h figures describe half a day.",
})
hlDayTruncated = promauto.NewGauge(prometheus.GaugeOpts{
Name: "hl_frontend_day_truncated_v2",
Help: "1 when the headline feed day covers less than 23 hours. The figures are still published: the honest answer to a truncated upstream is to say so, not to drop the day and let the window silently shorten.",
})
hlTruncatedDaysWindow = promauto.NewGauge(prometheus.GaugeOpts{
Name: "hl_frontend_truncated_days_window_v2",
Help: "Days inside the 30-day window whose published file stops short of 23 hours. Every windowed total is understated by roughly half a day for each one counted here.",
})

hlDataDay = promauto.NewGauge(prometheus.GaugeOpts{
Name: "hl_frontend_data_day_unix_v2",
Help: "UTC midnight (unix seconds) that opens the feed day the 24h gauges describe.",
Expand Down
Loading