diff --git a/harnesses/rpc-cost/cmd/script/blockers_test.go b/harnesses/rpc-cost/cmd/script/blockers_test.go new file mode 100644 index 000000000..80b384a96 --- /dev/null +++ b/harnesses/rpc-cost/cmd/script/blockers_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "math" + "strings" + "testing" +) + +// The two blockers the 2026-09-30 audit found and that survived until +// 2026-10-05. Both produced a plausible wrong number, which is why eighteen +// existing tests passed over them, and both moved a published winner. +// +// The audit asked for these as R5 and R7. + +// R5. Chainstack's archive surcharge is triggered by block age, not by method +// name, and several methods already carry the doubled figure in their own +// weight. Multiplying the weighted total again charged trace 3.6 RU against a +// real 2.0, an 80% overstatement. +func TestBlockAgeRaisesTheFloorItDoesNotScaleTheBill(t *testing.T) { + // Chainstack's real Ethereum weights: everything 1 RU, debug and trace 2. + p := Provider{ + Slug: "chainstack", + Currency: "USD", + Unit: "ru", + Weights: map[string]Weights{ + "ethereum": { + Default: f(1), + Methods: map[string]*float64{ + "eth_getLogs": f(1), + "eth_getBlockByNumber": f(1), + "eth_getBlockReceipts": f(1), + "debug_traceTransaction": f(2), + "trace_block": f(2), + }, + }, + }, + ArchiveRule: ArchiveRule{Kind: "block_age", Value: 2}, + } + + trace := Profile{ + ID: "trace", Chain: "ethereum", Archive: true, + Mix: map[string]float64{ + "debug_traceTransaction": 0.60, + "trace_block": 0.20, + "eth_getBlockByNumber": 0.20, + }, + } + got, err := unitsPerRequest(p, trace) + if err != nil { + t.Fatalf("trace: %v", err) + } + // Floor is default x 2 = 2. The two already-2 methods are unaffected by + // it; only getBlockByNumber is lifted, 1 -> 2. So a flat 2.0. + // + // The bug computed 0.6*2 + 0.2*2 + 0.2*1 = 1.8 and then doubled it to 3.6. + if math.Abs(got-2.0) > 1e-9 { + t.Errorf("trace units = %v, want 2.0 (3.6 is the double-count)", got) + } + + // The indexer profile must NOT move: every method in it is 1 RU, so the + // surcharge lifts all of them to 2 and the old multiplier reached the same + // answer. A fix that changed this number would be over-reaching. + indexer := Profile{ + ID: "indexer", Chain: "ethereum", Archive: true, + Mix: map[string]float64{ + "eth_getLogs": 0.50, + "eth_getBlockByNumber": 0.25, + "eth_getBlockReceipts": 0.25, + }, + } + got, err = unitsPerRequest(p, indexer) + if err != nil { + t.Fatalf("indexer: %v", err) + } + if math.Abs(got-2.0) > 1e-9 { + t.Errorf("indexer units = %v, want 2.0 unchanged", got) + } + + // And at tip the surcharge does not apply at all, so the listed weights + // stand: 0.6*2 + 0.2*2 + 0.2*1. + tip := trace + tip.Archive = false + got, err = unitsPerRequest(p, tip) + if err != nil { + t.Fatalf("tip: %v", err) + } + if math.Abs(got-1.8) > 1e-9 { + t.Errorf("non-archive trace units = %v, want 1.8", got) + } +} + +// A block_age rule with no chain default has nothing to act on. Inventing one +// would manufacture the number this bench exists to report, so it must refuse. +func TestBlockAgeWithoutADefaultRefusesToPrice(t *testing.T) { + p := Provider{ + Slug: "hypothetical", Currency: "USD", Unit: "ru", + Weights: map[string]Weights{ + "ethereum": {Methods: map[string]*float64{"eth_getLogs": f(1)}}, + }, + ArchiveRule: ArchiveRule{Kind: "block_age", Value: 2}, + } + pr := Profile{Chain: "ethereum", Archive: true, + Mix: map[string]float64{"eth_getLogs": 1.0}} + if _, err := unitsPerRequest(p, pr); err == nil { + t.Error("priced a block_age rule with no default weight; it must refuse") + } +} + +// R7. The dedicated cohort used to take needed = 0 for every plan, so the +// overage and max_units branches were unreachable. AWS AMB is dedicated +// capacity that ALSO bills per request, and it read $97.82 at every volume. +func TestDedicatedPlanThatMetersRequestsBillsThem(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + // AWS Managed Blockchain, us-east: a bc.t3.large node plus $3.00/1M + // requests, with nothing included. + p := Provider{Slug: "aws-amb", Currency: "USD", Unit: "request", Cohort: "dedicated"} + pl := Plan{ + ID: "bc-t3-large-us-east", Tier: "business", + MonthlyUSD: f(97.82), + IncludedUnits: nil, + OveragePer1M: f(3.00), + OverageAllowed: "true", + Chains: []string{"ethereum"}, + } + pr := Profile{Chain: "ethereum", Mix: map[string]float64{"eth_getBalance": 1.0}} + + q := quote(c, p, pl, pr, 1e9) + if !q.Eligible { + t.Fatalf("not eligible: %s", q.Reason) + } + // 1B requests at $3.00/1M is $3,000, on top of the node's $97.82. + if math.Abs(q.MonthlyUSD-3097.82) > 1e-6 { + t.Errorf("1B bill = %v, want 3097.82 (97.82 is the unmetered bug)", q.MonthlyUSD) + } + // And it must scale: the old behaviour was flat at every volume, which is + // what made it lead every Dedicated cell. + q10 := quote(c, p, pl, pr, 10e6) + if math.Abs(q10.MonthlyUSD-127.82) > 1e-6 { + t.Errorf("10M bill = %v, want 127.82", q10.MonthlyUSD) + } + if q10.MonthlyUSD >= q.MonthlyUSD { + t.Error("a metered plan must cost more at a higher volume") + } +} + +// The other 33 of 37 dedicated plans are genuinely flat: a monthly fee for +// unlimited requests. `included_units: null` means the opposite thing for +// them, and that ambiguity is what produced the blocker. They must not become +// ineligible for "allowance exceeded and no published overage rate". +func TestFlatDedicatedPlanStaysFlat(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + p := Provider{Slug: "getblock-dedicated", Currency: "USD", Unit: "request", Cohort: "dedicated"} + pl := Plan{ + ID: "dedicated-eth-full", Tier: "business", + MonthlyUSD: f(880), + Chains: []string{"ethereum"}, + } + pr := Profile{Chain: "ethereum", Mix: map[string]float64{"eth_getBalance": 1.0}} + + for _, requests := range []float64{10e6, 100e6, 1e9} { + q := quote(c, p, pl, pr, requests) + if !q.Eligible { + t.Fatalf("%.0f requests: not eligible: %s", requests, q.Reason) + } + if math.Abs(q.MonthlyUSD-880) > 1e-9 { + t.Errorf("%.0f requests: bill = %v, want a flat 880", requests, q.MonthlyUSD) + } + } +} + +// Zeeve publishes request ceilings with no overage rate. Past the ceiling the +// plan is unavailable, not free, and the old needed = 0 never reached this. +func TestDedicatedCeilingBindsAboveIt(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + p := Provider{Slug: "zeeve", Currency: "USD", Unit: "request", Cohort: "dedicated"} + pl := Plan{ + ID: "eth-basic", Tier: "growth", + MonthlyUSD: f(112), + IncludedUnits: f(250e6), + MaxUnits: f(250e6), + Chains: []string{"ethereum"}, + } + pr := Profile{Chain: "ethereum", Mix: map[string]float64{"eth_getBalance": 1.0}} + + // Inside the ceiling: the flat fee. + if q := quote(c, p, pl, pr, 100e6); !q.Eligible || math.Abs(q.MonthlyUSD-112) > 1e-9 { + t.Errorf("100M: eligible=%v bill=%v, want eligible at 112", q.Eligible, q.MonthlyUSD) + } + // Above it: unavailable, with the ceiling named. + q := quote(c, p, pl, pr, 1e9) + if q.Eligible { + t.Errorf("1B: priced at %v, but the plan's ceiling is 250M", q.MonthlyUSD) + } + if q.Reason == "" { + t.Error("ineligible with no reason; the page renders the reason") + } +} + +// R6. An unstated capability is not a granted one. +// +// `archive: null` and `trace: null` passed both gates, because one tested for +// the string "false" and the other for a non-nil pointer. 27 of the 106 usage +// plans are null on each field, and NOWNodes won trace cells on a capability +// it makes no claim about. +func TestUnstatedCapabilityIsNotRanked(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + p := Provider{ + Slug: "nownodes", Currency: "USD", Unit: "request", Cohort: "usage", + Weights: map[string]Weights{"ethereum": {Default: f(1)}}, + } + traceProfile := Profile{ + Chain: "ethereum", Archive: true, + Mix: map[string]float64{"debug_traceTransaction": 1.0}, + } + archiveProfile := Profile{ + Chain: "ethereum", Archive: true, + Mix: map[string]float64{"eth_getLogs": 1.0}, + } + + // Null on both: not ranked on either workload, and the reason must say the + // support is unpublished rather than absent, which is a claim we cannot + // make on the provider's behalf. + silent := Plan{ID: "enterprise", Tier: "enterprise", MonthlyUSD: f(200), + IncludedUnits: f(1e12), Archive: "", Trace: nil} + for name, pr := range map[string]Profile{"trace": traceProfile, "archive": archiveProfile} { + q := quote(c, p, silent, pr, 10e6) + if q.Eligible { + t.Errorf("%s: ranked at $%v on an unpublished capability", name, q.MonthlyUSD) + } + if !strings.Contains(q.Reason, "not published") { + t.Errorf("%s: reason %q should say the support is unpublished, not that "+ + "the plan lacks it", name, q.Reason) + } + } + + // An explicit claim still ranks, or the gate would empty the cells. + yes := Plan{ID: "pro", Tier: "growth", MonthlyUSD: f(90), + IncludedUnits: f(1e12), Archive: "true", Trace: boolp(true)} + if q := quote(c, p, yes, traceProfile, 10e6); !q.Eligible { + t.Errorf("explicit trace: true was not ranked: %s", q.Reason) + } + + // And an explicit denial keeps its own, different, reason. + no := Plan{ID: "start", Tier: "entry", MonthlyUSD: f(0), + IncludedUnits: f(1e12), Archive: "false", Trace: boolp(false)} + q := quote(c, p, no, archiveProfile, 10e6) + if q.Eligible { + t.Error("archive: false was ranked on an archive workload") + } + if strings.Contains(q.Reason, "not published") { + t.Errorf("reason %q: the provider DID state this, so it must not read as "+ + "a gap in our catalogue", q.Reason) + } +} + +func boolp(v bool) *bool { return &v } + +// The published winners at 10M and 100M sit EXACTLY on a Chainstack allowance, +// and that is worth pinning rather than discovering again. +// +// Chainstack's archive workloads cost 2 RU per request after the block_age +// floor, its tiers include 20M / 80M / 200M / 400M RU, and the bench's volume +// buckets are 10M / 100M / 1B requests. Round times round lands on round: the +// trace and indexer profiles at 10M requests need exactly 20,000,000 RU, which +// is exactly what Growth includes, and at 100M they need exactly 200,000,000, +// which is exactly Business. +// +// So the winner is real but maximally sensitive at precisely the volumes the +// page publishes: +1% of traffic moves trace@10M from 49 to 52 dollars, +50% +// moves it to 199. A test here means a change to the weights, the floor or the +// buckets cannot shift that edge silently. +func TestChainstackWinsOnAnExactAllowanceFit(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + p := Provider{ + Slug: "chainstack", Currency: "USD", Unit: "ru", Cohort: "usage", + Weights: map[string]Weights{"ethereum": { + Default: f(1), + Methods: map[string]*float64{ + "eth_getLogs": f(1), + "eth_getBlockByNumber": f(1), + "eth_getBlockReceipts": f(1), + "debug_traceTransaction": f(2), + "trace_block": f(2), + }, + }}, + ArchiveRule: ArchiveRule{Kind: "block_age", Value: 2}, + } + growth := Plan{ID: "growth", Tier: "growth", MonthlyUSD: f(49), + IncludedUnits: f(20e6), OveragePer1M: f(15.0), OverageAllowed: "true", + Archive: "true", Trace: boolp(true)} + business := Plan{ID: "business", Tier: "business", MonthlyUSD: f(499), + IncludedUnits: f(200e6), OveragePer1M: f(10.0), OverageAllowed: "true", + Archive: "true", Trace: boolp(true)} + + trace := Profile{ID: "trace", Chain: "ethereum", Archive: true, + Mix: map[string]float64{ + "debug_traceTransaction": 0.60, + "trace_block": 0.20, + "eth_getBlockByNumber": 0.20, + }} + + // Exactly at the allowance: the flat fee, no overage. + if q := quote(c, p, growth, trace, 10e6); !q.Eligible || math.Abs(q.MonthlyUSD-49) > 1e-9 { + t.Errorf("trace@10M = %v (eligible %v), want exactly 49", q.MonthlyUSD, q.Eligible) + } + if q := quote(c, p, business, trace, 100e6); !q.Eligible || math.Abs(q.MonthlyUSD-499) > 1e-9 { + t.Errorf("trace@100M = %v (eligible %v), want exactly 499", q.MonthlyUSD, q.Eligible) + } + + // One percent past it: the edge is real and the overage is charged. 10.1M + // requests cost 20.2M RU, so 0.2M RU over at 15 dollars per million. + q := quote(c, p, growth, trace, 10.1e6) + if math.Abs(q.MonthlyUSD-52) > 1e-6 { + t.Errorf("trace@10.1M = %v, want 52: the exact fit must not absorb real overage", + q.MonthlyUSD) + } + // And half again as much traffic is four times the bill. + q = quote(c, p, growth, trace, 15e6) + if math.Abs(q.MonthlyUSD-199) > 1e-6 { + t.Errorf("trace@15M = %v, want 199", q.MonthlyUSD) + } +} + +// The free tier can only be ranked after converting units to requests, and +// the conversion REVERSES the raw ranking. +// +// Comparing included_units across providers is meaningless: OnFinality bills +// 1 RU per request, dRPC 20 CU, Ankr 200 credits. By raw allowance Ankr looks +// second best in the cohort at 200M; by what it actually buys it is near the +// bottom. Each figure below was computed by hand from the catalogue first. +func TestFreeAllowanceConvertsAcrossUnits(t *testing.T) { + c := &Catalogue{FX: FX{EURUSD: 1}} + simple := Profile{ID: "simple-read", Chain: "ethereum", + Mix: map[string]float64{"eth_getBalance": 1.0}} + + cases := []struct { + name string + unit string + weight float64 + units float64 + period string + expected float64 + }{ + // 400,000 RU a DAY at 1 RU per request, over a 30-day month. + {"onfinality", "ru", 1, 400e3, "day", 12e6}, + // 210,000,000 CU a month at 20 CU per request. + {"drpc", "cu", 20, 210e6, "month", 10.5e6}, + // 200,000,000 credits a month at 200 per request: the inversion. + {"ankr", "credit", 200, 200e6, "month", 1e6}, + } + for _, tc := range cases { + p := Provider{ + Slug: tc.name, Currency: "USD", Unit: tc.unit, Cohort: "usage", + Weights: map[string]Weights{"ethereum": {Default: f(tc.weight)}}, + Plans: []Plan{{ + ID: "free", Tier: "free", MonthlyUSD: f(0), + IncludedUnits: f(tc.units), AllowancePeriod: tc.period, + }}, + } + got, plan, ok := c.FreeAllowanceRequests(p, simple) + if !ok { + t.Errorf("%s: no free allowance computed", tc.name) + continue + } + if plan != "free" { + t.Errorf("%s: plan = %q, want free", tc.name, plan) + } + if math.Abs(got-tc.expected) > 1 { + t.Errorf("%s: %.0f free requests/mo, want %.0f (%.0f %s at %.0f per request)", + tc.name, got, tc.expected, tc.units, tc.unit, tc.weight) + } + } + + // And the ordering that matters: by raw units Ankr beats OnFinality 500 to + // 1, by requests OnFinality beats Ankr 12 to 1. A bench that published the + // raw figure would rank them backwards. + if 200e6 <= 400e3 { + t.Fatal("premise wrong") + } + if 1e6 >= 12e6 { + t.Fatal("the inversion is the whole point of this metric") + } +} diff --git a/harnesses/rpc-cost/cmd/script/model.go b/harnesses/rpc-cost/cmd/script/model.go index fa2011da6..d16c87173 100644 --- a/harnesses/rpc-cost/cmd/script/model.go +++ b/harnesses/rpc-cost/cmd/script/model.go @@ -86,7 +86,24 @@ func unitsPerRequest(p Provider, pr Profile) (float64, error) { // documentary distinction, not a computational one, and reading // it as a no-op priced the indexer profile at 1 RU per request // instead of 2, publishing $3,990 for a bill of $8,990. - total *= p.ArchiveRule.Value + // + // It is NOT a multiplier on the total, which is how this was + // first written and what the 2026-09-30 audit caught. Several + // methods already carry the doubled figure in their own weight: + // debug_traceTransaction is listed at 2 RU and the empirical pass + // measured 2 RU even at tip-10, so the cost is method-driven + // there, not recency-driven. Multiplying again charged trace 3.6 + // RU against a real 2.0, an 80 % overstatement that moved the + // winner in two cells. + // + // Per method, the archive weight is therefore the larger of what + // the provider lists and what the block-age surcharge implies: + // a base read goes 1 -> 2, a debug call already at 2 stays 2. + var err error + total, err = blockAgeUnits(w, pr, p.ArchiveRule.Value) + if err != nil { + return 0, err + } case "none", "": // Either already encoded in the per-method weights (GetBlock // carries an independent archive column rather than a @@ -135,9 +152,16 @@ func quote(c *Catalogue, p Provider, pl Plan, pr Profile, requests float64) Quot return q } - unmetered := p.Cohort == "dedicated" - var upr float64 - if !unmetered { + // Every plan in the dedicated cohort bills in `request`, so one request + // is one unit and there are no per-method weights to resolve. It used to + // take upr = 0 and then needed = 0, which made the overage and max_units + // branches below unreachable for the whole cohort: AWS AMB read $97.82 at + // every volume against a real 1B bill of $3,097.82, and Zeeve's published + // ceilings bound nothing. Whether the requests are actually metered is + // decided further down, by the plan, not here by the cohort. + dedicated := p.Cohort == "dedicated" + upr := 1.0 + if !dedicated { var err error upr, err = unitsPerRequest(p, pr) if err != nil { @@ -150,19 +174,40 @@ func quote(c *Catalogue, p Provider, pl Plan, pr Profile, requests float64) Quot // Trace workloads on a plan that does not serve trace are not "more // expensive", they are unavailable. Same for archive. - if pr.Archive && pl.Archive == "false" { - q.Reason = "plan has no archive access" - return q + // + // And a plan that has never SAID must not be ranked either. Both fields + // used to pass when null — one test looked for the string "false", the + // other for a non-nil pointer — so 27 of the 106 usage plans were scored + // on archive and trace workloads without having claimed support. NOWNodes + // won trace cells that way, on a capability it makes no claim about. + // + // The reason distinguishes the two cases on purpose. "Does not serve" is + // the provider's statement; "has not published" is ours, and it is a gap + // in this catalogue rather than a limit of the product. Only the second is + // supportable for a null, and saying which it is turns each one into a + // research task instead of a silent omission. + if pr.Archive { + switch pl.Archive { + case "false": + q.Reason = "plan has no archive access" + return q + case "": + q.Reason = "archive support for this plan is not published; not ranked" + return q + } } - if isTraceProfile(pr) && pl.Trace != nil && !*pl.Trace { - q.Reason = "plan has no trace/debug access" - return q + if isTraceProfile(pr) { + if pl.Trace == nil { + q.Reason = "trace/debug support for this plan is not published; not ranked" + return q + } + if !*pl.Trace { + q.Reason = "plan has no trace/debug access" + return q + } } needed := requests * upr - if unmetered { - needed = 0 - } // Throughput is a second, independent meter, and for several // providers it binds before the bill does. Tatum and Moralis publish @@ -239,6 +284,18 @@ func quote(c *Catalogue, p Provider, pl Plan, pr Profile, requests float64) Quot included := 0.0 if pl.IncludedUnits != nil { included = *pl.IncludedUnits * periodScale(pl.AllowancePeriod) + } else if dedicated && pl.OveragePer1M == nil && len(pl.OverageBands) == 0 { + // A missing included_units means two opposite things in this cohort, + // and that ambiguity is what produced blocker 2. GetBlock's dedicated + // node serves unlimited requests for its monthly fee; AWS AMB includes + // none and bills every request on top. Both write null. + // + // The plan's own overage rate decides: publishing one means it meters + // from the first request, publishing none means the node is flat. 33 + // of the 37 plans here are flat and must stay so; the 4 that meter + // (AWS AMB in three regions, Shyft legacy-scale) now reach the branches + // below. + included = math.Inf(1) } // A daily cap is not a monthly pool. Infura's 15M credits/day cannot // be spent as 450M on the first of the month, so the plan is eligible @@ -454,6 +511,35 @@ func weightedUnits(w Weights, pr Profile) (float64, error) { return total, nil } +// blockAgeUnits prices a profile under a block-age archive surcharge. +// +// The surcharge raises the floor rather than scaling the bill: a method whose +// own listed weight is already at or above what the surcharge implies is +// unaffected by it. See the block_age branch in unitsPerRequest for why. +func blockAgeUnits(w Weights, pr Profile, mult float64) (float64, error) { + if w.Default == nil { + // Without a chain default there is nothing for the surcharge to act + // on, and guessing one would invent the number this bench exists to + // report. The provider's own archive table is the supported way to + // price this; see ArchiveWeights. + return 0, fmt.Errorf("block_age archive rule needs a chain default weight") + } + floor := *w.Default * mult + total, shares := 0.0, 0.0 + for method, share := range pr.Mix { + units, ok := w.Weight(method) + if !ok { + return 0, fmt.Errorf("no published unit cost for %s", method) + } + total += share * math.Max(units, floor) + shares += share + } + if shares > 0 { + total /= shares + } + return total, nil +} + // servesChain reports whether a plan is provisioned for a chain. An empty // list means the weights table decides, which is right for a per-request // plan and wrong for a dedicated node, so validate() requires the list on