From e54b60b25b201c8bc2e693b76592fc305417844a Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Fri, 15 May 2026 17:27:09 -0500 Subject: [PATCH 01/15] feat(load-test): capture and display load test results from benchmark runs Wire the load-test payload worker to write a JSON result file on graceful shutdown and surface it in the report UI. After the benchmark window ends the runner sends SIGINT to the load-test binary and continues proposing settlement blocks until the worker exits (up to 90s), giving the binary time to flush its result. The output path is stored as an artifact in RunResult and shown as a "Load test" link in the run list that opens a dedicated detail page. Also fixes an inverted slices.Contains check in the importer that caused required files to be silently skipped on error and optional files to abort the import. --- report/src/App.tsx | 5 + report/src/components/RunList.tsx | 21 +- report/src/pages/BenchmarkLoadTestDetail.tsx | 50 ++++ report/src/pages/LoadTestDetail.tsx | 228 +++++++++++-------- report/src/services/dataService.ts | 17 ++ report/src/types.ts | 4 + report/src/utils/useDataSeries.ts | 26 +++ runner/benchmark/result_metadata.go | 5 +- runner/clients/baserethnode/client.go | 4 + runner/clients/builder/client.go | 9 + runner/clients/common/proxy/proxy.go | 77 ++++++- runner/clients/common/proxy/proxy_test.go | 104 +++++++++ runner/clients/geth/client.go | 4 + runner/clients/reth/client.go | 4 + runner/clients/types/types.go | 3 + runner/importer/service.go | 3 +- runner/network/network_benchmark.go | 11 + runner/network/sequencer_benchmark.go | 70 +++++- runner/network/types/types.go | 4 + runner/payload/factory.go | 2 +- runner/payload/loadtest/load_test_worker.go | 147 +++++++++--- runner/payload/txfuzz/tx_fuzz_worker.go | 3 +- runner/payload/worker/types.go | 7 + runner/service.go | 13 +- 24 files changed, 672 insertions(+), 149 deletions(-) create mode 100644 report/src/pages/BenchmarkLoadTestDetail.tsx create mode 100644 runner/clients/common/proxy/proxy_test.go diff --git a/report/src/App.tsx b/report/src/App.tsx index cef213a2..a1050e76 100644 --- a/report/src/App.tsx +++ b/report/src/App.tsx @@ -5,6 +5,7 @@ import RedirectToLatestRun from "./pages/RedirectToLatestRun"; import LoadTestLanding from "./pages/LoadTestLanding"; import LoadTestAllRuns from "./pages/LoadTestAllRuns"; import LoadTestDetail from "./pages/LoadTestDetail"; +import BenchmarkLoadTestDetail from "./pages/BenchmarkLoadTestDetail"; import ErrorBoundary from "./components/ErrorBoundary"; function App() { @@ -22,6 +23,10 @@ function App() { path="/load-tests/:network/:timestamp" element={} /> + } + /> } /> - +
+ + {run.result?.artifacts?.loadTestResult && ( + + Load test + + )} +
{Object.keys(COLUMN_DEFINITIONS).map((column) => ( { + const { outputDir } = useParams(); + const { + data: result, + isLoading, + error, + } = useBenchmarkLoadTestResult(outputDir); + + return ( +
+ +
+ {isLoading && ( +
+ Loading benchmark load test… +
+ )} + + {error && ( +
+ Failed to load benchmark load test result: {String(error)} +
+ )} + + {result && ( + + Output dir: {outputDir} + + } + backLink={{ + to: "/latest", + label: "View benchmark runs →", + }} + /> + )} +
+
+ ); +}; + +export default BenchmarkLoadTestDetail; diff --git a/report/src/pages/LoadTestDetail.tsx b/report/src/pages/LoadTestDetail.tsx index fd5050c8..d902355f 100644 --- a/report/src/pages/LoadTestDetail.tsx +++ b/report/src/pages/LoadTestDetail.tsx @@ -1,5 +1,5 @@ import { Link, useParams } from "react-router-dom"; -import { useMemo } from "react"; +import { type ReactNode, useMemo } from "react"; import Navbar from "../components/Navbar"; import StatCard, { Stat, StatGrid } from "../components/StatCard"; import PercentileBarChart, { @@ -149,50 +149,127 @@ const SummarySection = ({ result }: { result: LoadTestResult }) => { ); }; -const LoadTestDetail = () => { - const { network, timestamp } = useParams(); - const { - data: result, - isLoading, - error, - } = useLoadTestResult(network, timestamp); +interface LoadTestReportContentProps { + result: LoadTestResult; + title: string; + subtitle: ReactNode; + backLink?: { + to: string; + label: string; + }; +} +export const LoadTestReportContent = ({ + result, + title, + subtitle, + backLink, +}: LoadTestReportContentProps) => { const blockLatencyRows = useMemo( - () => (result ? buildLatencyRows(result.block_latency) : []), + () => buildLatencyRows(result.block_latency), [result], ); const flashblocksLatencyRows = useMemo( - () => (result ? buildLatencyRows(result.flashblocks_latency) : []), + () => buildLatencyRows(result.flashblocks_latency), [result], ); return ( -
- -
-
-
+ <> +
+
+ {backLink && ( - View all runs → + {backLink.label} -

- {timestamp ? formatLoadTestTimestamp(timestamp) : "Load test"} -

-

- Network: {network} - {timestamp && ( - <> - {" · "} - {timestamp} - - )} -

-
-
+ )} +

+ {title} +

+

{subtitle}

+
+
+ + + + {result.throughput_timeseries && + result.throughput_timeseries.length > 1 && ( + + + + )} + + {result.config && } + + + + + + + + + + + + + {(() => { + const reverted = result.throughput.total_reverted; + const reasons: [string, number][] = [ + ...(reverted > 0 + ? [["reverted", reverted] as [string, number]] + : []), + ...result.top_failure_reasons, + ]; + if (reasons.length === 0) { + return ( +
+ No failures recorded. +
+ ); + } + return ( +
    + {reasons.map(([reason, count]) => ( +
  • + {reason} + {count.toLocaleString()} +
    • + ))} +
+ ); + })()} + + + ); +}; +const LoadTestDetail = () => { + const { network, timestamp } = useParams(); + const { + data: result, + isLoading, + error, + } = useLoadTestResult(network, timestamp); + + return ( +
+ +
{isLoading && (
Loading load test…
)} @@ -204,74 +281,27 @@ const LoadTestDetail = () => { )} {result && ( - <> - - - {result.throughput_timeseries && - result.throughput_timeseries.length > 1 && ( - - - - )} - - {result.config && } - - - - - - - - - - - - - {(() => { - const reverted = result.throughput.total_reverted; - const reasons: [string, number][] = [ - ...(reverted > 0 - ? [["reverted", reverted] as [string, number]] - : []), - ...result.top_failure_reasons, - ]; - if (reasons.length === 0) { - return ( -
- No failures recorded. -
- ); - } - return ( -
    - {reasons.map(([reason, count]) => ( -
  • - {reason} - - {count.toLocaleString()} - -
    • - ))} -
- ); - })()} - - + + Network: {network} + {timestamp && ( + <> + {" · "} + + {timestamp} + + + )} + + } + backLink={{ + to: `/load-tests/${network ?? "sepolia"}/all`, + label: "View all runs →", + }} + /> )}
diff --git a/report/src/services/dataService.ts b/report/src/services/dataService.ts index 458c4b66..45236ac5 100644 --- a/report/src/services/dataService.ts +++ b/report/src/services/dataService.ts @@ -79,6 +79,23 @@ export class DataService { return await response.json(); } + + async getBenchmarkLoadTestResult( + outputDir: string, + artifactPath = "load-test-result.json", + ): Promise { + const response = await fetch( + `${this.baseUrl}output/${encodeURIComponent(outputDir)}/${encodeURIComponent(artifactPath)}`, + ); + + if (!response.ok) { + throw new Error( + `Failed to fetch benchmark load test result: ${response.status} ${response.statusText}`, + ); + } + + return await response.json(); + } } // Configuration helper to determine base URL from environment diff --git a/report/src/types.ts b/report/src/types.ts index 13d8c212..ee3f9ffa 100644 --- a/report/src/types.ts +++ b/report/src/types.ts @@ -87,6 +87,10 @@ export interface BenchmarkRun { gasPerSecond: number; newPayload: number; }; + artifacts?: { + loadTestResult?: string; + [key: string]: string | undefined; + }; } | null; } diff --git a/report/src/utils/useDataSeries.ts b/report/src/utils/useDataSeries.ts index 24f08eca..5f36a7b7 100644 --- a/report/src/utils/useDataSeries.ts +++ b/report/src/utils/useDataSeries.ts @@ -139,3 +139,29 @@ export const useLoadTestResult = ( }, ); }; + +export const useBenchmarkLoadTestResult = ( + outputDir: string | undefined, + artifactPath?: string, +) => { + const fetcher = useCallback(async (): Promise => { + if (!outputDir) { + throw new Error("outputDir required"); + } + const dataService = getDataService(); + return await dataService.getBenchmarkLoadTestResult(outputDir, artifactPath); + }, [outputDir, artifactPath]); + + return useSWR( + outputDir + ? `benchmark-load-test-${outputDir}-${artifactPath ?? "load-test-result.json"}` + : null, + fetcher, + { + dedupingInterval: 12 * 60 * 60 * 1000, + revalidateOnFocus: false, + errorRetryCount: 3, + errorRetryInterval: 5000, + }, + ); +}; diff --git a/runner/benchmark/result_metadata.go b/runner/benchmark/result_metadata.go index 22f79e6f..641560be 100644 --- a/runner/benchmark/result_metadata.go +++ b/runner/benchmark/result_metadata.go @@ -12,6 +12,7 @@ type RunResult struct { SequencerMetrics *types.SequencerKeyMetrics `json:"sequencerMetrics,omitempty"` ValidatorMetrics *types.ValidatorKeyMetrics `json:"validatorMetrics,omitempty"` ClientVersion string `json:"clientVersion,omitempty"` + Artifacts map[string]string `json:"artifacts,omitempty"` } // MachineInfo contains information about the machine running the benchmark @@ -51,7 +52,9 @@ func (runs *RunGroup) AddResult(testIdx int, runResult RunResult) { } const ( - BenchmarkRunTag = "BenchmarkRun" + BenchmarkRunTag = "BenchmarkRun" + LoadTestResultArtifactKey = "loadTestResult" + LoadTestResultFileName = "load-test-result.json" ) func RunGroupFromTestPlans(testPlans []TestPlan, machineInfo *MachineInfo) RunGroup { diff --git a/runner/clients/baserethnode/client.go b/runner/clients/baserethnode/client.go index 844d33b3..3a84cff4 100644 --- a/runner/clients/baserethnode/client.go +++ b/runner/clients/baserethnode/client.go @@ -276,3 +276,7 @@ func (r *BaseRethNodeClient) FlashblocksClient() types.FlashblocksClient { func (r *BaseRethNodeClient) SupportsFlashblocks() bool { return true } + +func (r *BaseRethNodeClient) FlashblocksWsURL() string { + return "" +} diff --git a/runner/clients/builder/client.go b/runner/clients/builder/client.go index b229905c..605f93e1 100644 --- a/runner/clients/builder/client.go +++ b/runner/clients/builder/client.go @@ -128,3 +128,12 @@ func (r *BuilderClient) FlashblocksClient() types.FlashblocksClient { func (r *BuilderClient) SupportsFlashblocks() bool { return false } + +// FlashblocksWsURL returns the local WebSocket URL of the flashblocks server +// hosted by the builder. +func (r *BuilderClient) FlashblocksWsURL() string { + if r.websocketPort == 0 { + return "" + } + return fmt.Sprintf("ws://localhost:%d", r.websocketPort) +} diff --git a/runner/clients/common/proxy/proxy.go b/runner/clients/common/proxy/proxy.go index f261c39e..b212ec0b 100644 --- a/runner/clients/common/proxy/proxy.go +++ b/runner/clients/common/proxy/proxy.go @@ -17,6 +17,7 @@ import ( "fmt" "io" "net/http" + "sync" "github.com/base/base-bench/runner/network/mempool" "github.com/ethereum/go-ethereum/common" @@ -31,6 +32,7 @@ type ProxyServer struct { pendingTxs []*ethTypes.Transaction clientURL string mempool *mempool.StaticWorkloadMempool + mu sync.Mutex } func NewProxyServer(clientURL string, log log.Logger, port int, mempool *mempool.StaticWorkloadMempool) *ProxyServer { @@ -62,11 +64,29 @@ func (p *ProxyServer) Run(ctx context.Context) error { } func (p *ProxyServer) PendingTxs() []*ethTypes.Transaction { - return p.pendingTxs + p.mu.Lock() + defer p.mu.Unlock() + + txs := make([]*ethTypes.Transaction, len(p.pendingTxs)) + copy(txs, p.pendingTxs) + return txs } func (p *ProxyServer) ClearPendingTxs() { + p.mu.Lock() + defer p.mu.Unlock() + + p.pendingTxs = make([]*ethTypes.Transaction, 0) +} + +func (p *ProxyServer) DrainPendingTxs() []*ethTypes.Transaction { + p.mu.Lock() + defer p.mu.Unlock() + + txs := make([]*ethTypes.Transaction, len(p.pendingTxs)) + copy(txs, p.pendingTxs) p.pendingTxs = make([]*ethTypes.Transaction, 0) + return txs } // Stop stops both the proxy server and the underlying client @@ -89,13 +109,20 @@ func (p *ProxyServer) handleRequest(w http.ResponseWriter, r *http.Request) { return } - var request struct { + type rpcRequest struct { Method string `json:"method"` Params json.RawMessage `json:"params"` ID interface{} `json:"id"` JSONRPC string `json:"jsonrpc"` } + if len(body) > 0 && body[0] == '[' { + p.handleBatchRequest(w, body) + return + } + + var request rpcRequest + if err := json.Unmarshal(body, &request); err != nil { http.Error(w, "Error parsing request", http.StatusBadRequest) return @@ -164,6 +191,50 @@ func (p *ProxyServer) handleRequest(w http.ResponseWriter, r *http.Request) { p.DebugResponse(request.Method, request.Params, respBody) } +func (p *ProxyServer) handleBatchRequest(w http.ResponseWriter, body []byte) { + type rpcRequest struct { + Method string `json:"method"` + Params json.RawMessage `json:"params"` + ID interface{} `json:"id"` + JSONRPC string `json:"jsonrpc"` + } + + var requests []rpcRequest + if err := json.Unmarshal(body, &requests); err != nil { + http.Error(w, "Error parsing batch request", http.StatusBadRequest) + return + } + + type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error interface{} `json:"error,omitempty"` + } + + responses := make([]rpcResponse, 0, len(requests)) + for _, request := range requests { + handled, result, err := p.OverrideRequest(request.Method, request.Params) + response := rpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + } + if err != nil { + response.Error = map[string]interface{}{"code": -32000, "message": err.Error()} + } else if handled { + response.Result = result + } else { + response.Error = map[string]interface{}{"code": -32601, "message": "method not supported in proxy batch mode"} + } + responses = append(responses, response) + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(responses); err != nil { + p.log.Error("Error encoding batch response", "err", err) + } +} + func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) (bool, json.RawMessage, error) { switch method { case "eth_getTransactionCount": @@ -204,7 +275,9 @@ func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) return false, nil, fmt.Errorf("failed to decode transaction: %w", err) } + p.mu.Lock() p.pendingTxs = append(p.pendingTxs, &tx) + p.mu.Unlock() txHash := tx.Hash().Hex() jsonResponse, _ := json.Marshal(txHash) diff --git a/runner/clients/common/proxy/proxy_test.go b/runner/clients/common/proxy/proxy_test.go new file mode 100644 index 00000000..a5c0678f --- /dev/null +++ b/runner/clients/common/proxy/proxy_test.go @@ -0,0 +1,104 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "math/big" + "net/http" + "net/http/httptest" + "testing" + + "github.com/base/base-bench/runner/network/mempool" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" +) + +func TestHandleBatchRequestCapturesRawTransactions(t *testing.T) { + chainID := big.NewInt(8453) + tx := signedTestTx(t, chainID) + rawTx, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal tx: %v", err) + } + + server := NewProxyServer( + "http://127.0.0.1:8545", + log.New(), + 0, + mempool.NewStaticWorkloadMempool(log.New(), chainID), + ) + + body, err := json.Marshal([]map[string]any{ + { + "jsonrpc": "2.0", + "id": 0, + "method": "eth_sendRawTransaction", + "params": []string{hexutil.Encode(rawTx)}, + }, + }) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + rec := httptest.NewRecorder() + + server.handleRequest(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", rec.Code, rec.Body.String()) + } + + var responses []struct { + Result string `json:"result"` + Error map[string]any `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &responses); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(responses) != 1 { + t.Fatalf("expected 1 response, got %d", len(responses)) + } + if responses[0].Error != nil { + t.Fatalf("expected successful response, got error %v", responses[0].Error) + } + if responses[0].Result != tx.Hash().Hex() { + t.Fatalf("expected tx hash %s, got %s", tx.Hash().Hex(), responses[0].Result) + } + + pending := server.DrainPendingTxs() + if len(pending) != 1 { + t.Fatalf("expected 1 pending tx, got %d", len(pending)) + } + if pending[0].Hash() != tx.Hash() { + t.Fatalf("expected pending tx %s, got %s", tx.Hash(), pending[0].Hash()) + } +} + +func signedTestTx(t *testing.T, chainID *big.Int) *types.Transaction { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tx := types.NewTx(&types.DynamicFeeTx{ + ChainID: chainID, + Nonce: 0, + GasTipCap: big.NewInt(1), + GasFeeCap: big.NewInt(1), + Gas: 21_000, + To: &common.Address{1}, + Value: big.NewInt(1), + }) + + signed, err := types.SignTx(tx, types.NewIsthmusSigner(chainID), key) + if err != nil { + t.Fatalf("sign tx: %v", err) + } + return signed +} diff --git a/runner/clients/geth/client.go b/runner/clients/geth/client.go index 89e40b67..a4efd25b 100644 --- a/runner/clients/geth/client.go +++ b/runner/clients/geth/client.go @@ -280,3 +280,7 @@ func (g *GethClient) FlashblocksClient() types.FlashblocksClient { func (g *GethClient) SupportsFlashblocks() bool { return false } + +func (g *GethClient) FlashblocksWsURL() string { + return "" +} diff --git a/runner/clients/reth/client.go b/runner/clients/reth/client.go index 7a4a25c8..35739d4e 100644 --- a/runner/clients/reth/client.go +++ b/runner/clients/reth/client.go @@ -296,3 +296,7 @@ func (r *RethClient) FlashblocksClient() types.FlashblocksClient { func (r *RethClient) SupportsFlashblocks() bool { return true } + +func (r *RethClient) FlashblocksWsURL() string { + return "" +} diff --git a/runner/clients/types/types.go b/runner/clients/types/types.go index a0be0080..c88b6212 100644 --- a/runner/clients/types/types.go +++ b/runner/clients/types/types.go @@ -32,4 +32,7 @@ type ExecutionClient interface { SetHead(ctx context.Context, blockNumber uint64) error FlashblocksClient() FlashblocksClient // returns nil for clients that don't support flashblocks SupportsFlashblocks() bool // returns true if the client supports receiving flashblock payloads + // FlashblocksWsURL returns the local WebSocket URL hosted by this client, + // or an empty string when the client does not expose one. + FlashblocksWsURL() string } diff --git a/runner/importer/service.go b/runner/importer/service.go index e771abb6..1ed2008d 100644 --- a/runner/importer/service.go +++ b/runner/importer/service.go @@ -75,6 +75,7 @@ func (s *Service) downloadOutputFiles(baseURL, runID, runOutputDir string) error "result-sequencer.json", "metrics-validator.json", "metrics-sequencer.json", + benchmark.LoadTestResultFileName, } requiredFiles := []string{ @@ -97,7 +98,7 @@ func (s *Service) downloadOutputFiles(baseURL, runID, runOutputDir string) error // Try to download the file err := s.downloadFile(fileURL, localFilePath) if err != nil { - if !slices.Contains(requiredFiles, fileName) { + if slices.Contains(requiredFiles, fileName) { return errors.Wrap(err, "failed to download file") } s.log.Warn("Failed to download file (continuing)", "file", fileName, "url", fileURL, "error", err) diff --git a/runner/network/network_benchmark.go b/runner/network/network_benchmark.go index 37b336c7..a9b41ee7 100644 --- a/runner/network/network_benchmark.go +++ b/runner/network/network_benchmark.go @@ -279,6 +279,17 @@ func (nb *NetworkBenchmark) GetResult() (*benchmark.RunResult, error) { result.ValidatorMetrics = nb.collectedValidatorMetrics } + artifacts := make(map[string]string) + if nb.testConfig.LoadTestOutputPath != "" { + if _, err := os.Stat(nb.testConfig.LoadTestOutputPath); err == nil { + artifacts[benchmark.LoadTestResultArtifactKey] = benchmark.LoadTestResultFileName + } + } + if len(artifacts) == 0 { + artifacts = nil + } + result.Artifacts = artifacts + return result, nil } diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index 29aa8c28..1a7e790d 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -15,6 +15,7 @@ import ( "github.com/base/base-bench/runner/network/proofprogram/fakel1" benchtypes "github.com/base/base-bench/runner/network/types" "github.com/base/base-bench/runner/payload" + payloadworker "github.com/base/base-bench/runner/payload/worker" "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" @@ -24,6 +25,8 @@ import ( "github.com/pkg/errors" ) +const gracefulWorkerShutdownTimeout = 90 * time.Second + type sequencerBenchmark struct { log log.Logger sequencerClient types.ExecutionClient @@ -282,8 +285,13 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. payloads = append(payloads, *payload) } - err = consensusClient.Stop(benchmarkCtx) - if err != nil { + pendingTxs, shutdownErr := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs) + if shutdownErr != nil { + errChan <- shutdownErr + return + } + + if err := consensusClient.Stop(benchmarkCtx); err != nil { nb.log.Warn("failed to stop consensus client", "err", err) } @@ -309,6 +317,64 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. } } +func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( + ctx context.Context, + transactionWorker payloadworker.Worker, + consensusClient *consensus.SequencerConsensusClient, + pendingTxs int, +) (int, error) { + gracefulWorker, ok := transactionWorker.(payloadworker.GracefulShutdownWorker) + if !ok { + return pendingTxs, nil + } + + if err := gracefulWorker.BeginGracefulShutdown(ctx); err != nil { + return pendingTxs, errors.Wrap(err, "failed to begin graceful payload worker shutdown") + } + + timeout := time.NewTimer(gracefulWorkerShutdownTimeout) + defer timeout.Stop() + + settlementBlock := 0 + for { + select { + case <-gracefulWorker.Done(): + nb.log.Info("Payload worker stopped gracefully", "settlement_blocks", settlementBlock) + return pendingTxs, nil + case <-timeout.C: + nb.log.Warn("Timed out waiting for payload worker to stop gracefully", "settlement_blocks", settlementBlock) + return pendingTxs, nil + default: + } + + blockMetrics := metrics.NewBlockMetrics() + txsSent, err := transactionWorker.SendTxs(ctx, pendingTxs) + if err != nil { + return pendingTxs, errors.Wrap(err, "failed to collect settlement transactions") + } + + payload, err := consensusClient.Propose(ctx, blockMetrics, true) + if err != nil { + return pendingTxs, errors.Wrap(err, "failed to propose settlement block") + } + if payload == nil { + return pendingTxs, errors.New("received nil settlement payload from consensus client") + } + + userTxsIncluded := len(payload.Transactions) - 1 + if userTxsIncluded < 0 { + userTxsIncluded = 0 + } + pendingTxs = pendingTxs + txsSent - userTxsIncluded + if pendingTxs < 0 { + pendingTxs = 0 + } + + settlementBlock++ + time.Sleep(nb.config.Params.BlockTime) + } +} + // flashblockCollector implements FlashblockListener to collect flashblocks. type flashblockCollector struct { log log.Logger diff --git a/runner/network/types/types.go b/runner/network/types/types.go index c219c72a..2b93c1e5 100644 --- a/runner/network/types/types.go +++ b/runner/network/types/types.go @@ -40,6 +40,10 @@ type TestConfig struct { PrefundPrivateKey ecdsa.PrivateKey PrefundAmount big.Int + + // LoadTestOutputPath is the optional JSON summary path used by the + // load-test payload worker. + LoadTestOutputPath string } // BatcherAddr returns the batcher address, computing it if necessary diff --git a/runner/payload/factory.go b/runner/payload/factory.go index b8e7176e..7894563f 100644 --- a/runner/payload/factory.go +++ b/runner/payload/factory.go @@ -38,7 +38,7 @@ func NewPayloadWorker(ctx context.Context, log log.Logger, testConfig *benchtype def = &loadtest.LoadTestPayloadDefinition{} } worker, err = loadtest.NewLoadTestPayloadWorker( - log, sequencerClient.ClientURL(), params, privateKey, amount, config, genesis.Config.ChainID, *def) + log, sequencerClient.ClientURL(), sequencerClient.FlashblocksWsURL(), params, privateKey, amount, config, genesis.Config.ChainID, *def, testConfig.LoadTestOutputPath) case "transfer-only": worker, err = transferonly.NewTransferPayloadWorker( ctx, log, sequencerClient.ClientURL(), params, privateKey, amount, &genesis, definition.Params) diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index 5b6d62ff..0723496c 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -9,6 +9,9 @@ import ( "math/big" "os" "os/exec" + "path/filepath" + "sync" + "time" "github.com/base/base-bench/runner/clients/common/proxy" "github.com/base/base-bench/runner/config" @@ -32,13 +35,16 @@ type LoadTestPayloadDefinition struct { // loadTestConfig is the YAML config written to a temp file for the load-test binary. type loadTestConfig struct { - RPC string `yaml:"rpc"` - SenderCount uint64 `yaml:"sender_count"` - TargetGPS uint64 `yaml:"target_gps"` - Duration string `yaml:"duration"` - Seed uint64 `yaml:"seed"` - FundingAmount string `yaml:"funding_amount"` - Transactions yaml.Node `yaml:"transactions"` + RPC string `yaml:"rpc,omitempty"` + TransactionSubmissionRPCs []string `yaml:"transaction_submission_rpcs"` + QueryRPC string `yaml:"query_rpc"` + FlashblocksWs string `yaml:"flashblocks_ws"` + SenderCount uint64 `yaml:"sender_count"` + TargetGPS uint64 `yaml:"target_gps"` + Duration string `yaml:"duration"` + Seed uint64 `yaml:"seed"` + FundingAmount string `yaml:"funding_amount"` + Transactions yaml.Node `yaml:"transactions"` } type loadTestPayloadWorker struct { @@ -46,13 +52,19 @@ type loadTestPayloadWorker struct { prefundSK string loadTestBin string elRPCURL string + flashblocksURL string gasLimit uint64 blockTimeSec uint64 params LoadTestPayloadDefinition mempool *mempool.StaticWorkloadMempool proxyServer *proxy.ProxyServer cmd *exec.Cmd + done chan struct{} + waitErr error + waitMu sync.Mutex + shutdownOnce sync.Once configFilePath string + outputPath string } // NewLoadTestPayloadWorker creates a worker that runs the base-load-test binary @@ -60,12 +72,14 @@ type loadTestPayloadWorker struct { func NewLoadTestPayloadWorker( log log.Logger, elRPCURL string, + flashblocksURL string, params types.RunParams, prefundedPrivateKey ecdsa.PrivateKey, prefundAmount *big.Int, cfg config.Config, chainID *big.Int, definition LoadTestPayloadDefinition, + outputPath string, ) (worker.Worker, error) { mp := mempool.NewStaticWorkloadMempool(log, chainID) ps := proxy.NewProxyServer(elRPCURL, log, cfg.ProxyPort(), mp) @@ -76,15 +90,17 @@ func NewLoadTestPayloadWorker( } w := &loadTestPayloadWorker{ - log: log, - prefundSK: hex.EncodeToString(prefundedPrivateKey.D.Bytes()), - loadTestBin: cfg.LoadTestBinary(), - elRPCURL: elRPCURL, - gasLimit: params.GasLimit, - blockTimeSec: blockTimeSec, - params: definition, - mempool: mp, - proxyServer: ps, + log: log, + prefundSK: hex.EncodeToString(prefundedPrivateKey.D.Bytes()), + loadTestBin: cfg.LoadTestBinary(), + elRPCURL: elRPCURL, + flashblocksURL: flashblocksURL, + gasLimit: params.GasLimit, + blockTimeSec: blockTimeSec, + params: definition, + mempool: mp, + proxyServer: ps, + outputPath: outputPath, } return w, nil @@ -111,23 +127,85 @@ func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout cmd.Env = append(os.Environ(), fmt.Sprintf("FUNDER_KEY=%s", w.prefundSK)) + if w.outputPath != "" { + if err := os.MkdirAll(filepath.Dir(w.outputPath), 0755); err != nil { + return errors.Wrap(err, "failed to create load-test output directory") + } + cmd.Env = append(cmd.Env, fmt.Sprintf("LOAD_TEST_OUTPUT=%s", w.outputPath)) + } if err := cmd.Start(); err != nil { return errors.Wrap(err, "failed to start load test binary") } w.cmd = cmd + w.done = make(chan struct{}) + go func() { + err := cmd.Wait() + w.waitMu.Lock() + w.waitErr = err + w.waitMu.Unlock() + close(w.done) + }() return nil } +func (w *loadTestPayloadWorker) BeginGracefulShutdown(ctx context.Context) error { + if w.cmd == nil || w.cmd.Process == nil { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-w.Done(): + return nil + default: + } + + var signalErr error + w.shutdownOnce.Do(func() { + w.log.Info("Stopping load test process gracefully", "pid", w.cmd.Process.Pid, "output", w.outputPath) + signalErr = w.cmd.Process.Signal(os.Interrupt) + }) + if signalErr != nil { + select { + case <-w.Done(): + return nil + default: + } + } + return signalErr +} + +func (w *loadTestPayloadWorker) Done() <-chan struct{} { + if w.done != nil { + return w.done + } + + done := make(chan struct{}) + close(done) + return done +} + func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { if w.cmd != nil && w.cmd.Process != nil { - w.log.Info("Stopping load test process", "pid", w.cmd.Process.Pid) - if err := w.cmd.Process.Kill(); err != nil { - w.log.Warn("failed to kill load test process", "err", err) - } else { - // Reap the process to avoid zombies. - _, _ = w.cmd.Process.Wait() + if err := w.BeginGracefulShutdown(ctx); err != nil { + w.log.Warn("failed to signal load test process", "err", err) + } + + select { + case <-w.Done(): + case <-time.After(10 * time.Second): + w.log.Warn("load test process did not stop gracefully, killing", "pid", w.cmd.Process.Pid) + if err := w.cmd.Process.Kill(); err != nil { + w.log.Warn("failed to kill load test process", "err", err) + } + select { + case <-w.Done(): + case <-time.After(5 * time.Second): + w.log.Warn("timed out waiting for killed load test process") + } } } @@ -144,8 +222,7 @@ func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { func (w *loadTestPayloadWorker) SendTxs(ctx context.Context, _ int) (int, error) { w.log.Info("Collecting txs from load test") - pendingTxs := w.proxyServer.PendingTxs() - w.proxyServer.ClearPendingTxs() + pendingTxs := w.proxyServer.DrainPendingTxs() w.mempool.AddTransactions(pendingTxs) return len(pendingTxs), nil @@ -206,14 +283,22 @@ func (w *loadTestPayloadWorker) writeConfig() (string, error) { transactions = defaultTransactions() } + flashblocksURL := w.flashblocksURL + if flashblocksURL == "" { + flashblocksURL = "ws://localhost:7111" + } + config := loadTestConfig{ - RPC: w.proxyServer.ClientURL(), - SenderCount: senderCount, - TargetGPS: targetGPS, - Duration: "99999s", - Seed: randomSeed(), - FundingAmount: fundingAmount, - Transactions: transactions, + RPC: w.proxyServer.ClientURL(), + TransactionSubmissionRPCs: []string{w.proxyServer.ClientURL()}, + QueryRPC: w.proxyServer.ClientURL(), + FlashblocksWs: flashblocksURL, + SenderCount: senderCount, + TargetGPS: targetGPS, + Duration: "99999s", + Seed: randomSeed(), + FundingAmount: fundingAmount, + Transactions: transactions, } data, err := yaml.Marshal(&config) diff --git a/runner/payload/txfuzz/tx_fuzz_worker.go b/runner/payload/txfuzz/tx_fuzz_worker.go index b5167d42..87fd5010 100644 --- a/runner/payload/txfuzz/tx_fuzz_worker.go +++ b/runner/payload/txfuzz/tx_fuzz_worker.go @@ -84,8 +84,7 @@ func (t *txFuzzPayloadWorker) Stop(ctx context.Context) error { func (t *txFuzzPayloadWorker) SendTxs(ctx context.Context, _ int) (int, error) { t.log.Info("Sending txs in tx-fuzz mode") - pending := t.proxyServer.PendingTxs() - t.proxyServer.ClearPendingTxs() + pending := t.proxyServer.DrainPendingTxs() t.mempool.AddTransactions(pending) return len(pending), nil diff --git a/runner/payload/worker/types.go b/runner/payload/worker/types.go index 69906ffe..a0e05725 100644 --- a/runner/payload/worker/types.go +++ b/runner/payload/worker/types.go @@ -18,3 +18,10 @@ type Worker interface { Stop(ctx context.Context) error Mempool() mempool.FakeMempool } + +// GracefulShutdownWorker can stop generating transactions while the benchmark +// sequencer keeps producing settlement blocks. +type GracefulShutdownWorker interface { + BeginGracefulShutdown(ctx context.Context) error + Done() <-chan struct{} +} diff --git a/runner/service.go b/runner/service.go index 2d7f64fa..2add585f 100644 --- a/runner/service.go +++ b/runner/service.go @@ -428,12 +428,13 @@ func (s *service) runTest(ctx context.Context, params types.RunParams, workingDi prefundAmount := new(big.Int).Mul(big.NewInt(1e6), big.NewInt(ethparams.Ether)) config := &types.TestConfig{ - Params: params, - Config: s.config, - Genesis: *genesis, - BatcherKey: *batcherKey, - PrefundPrivateKey: *prefundKey, - PrefundAmount: *prefundAmount, + Params: params, + Config: s.config, + Genesis: *genesis, + BatcherKey: *batcherKey, + PrefundPrivateKey: *prefundKey, + PrefundAmount: *prefundAmount, + LoadTestOutputPath: path.Join(outputDir, benchmark.LoadTestResultFileName), } // Run benchmark From edf0c3e3b959931cb46abfa8e2a4cea4c6391be6 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Fri, 15 May 2026 20:46:20 -0500 Subject: [PATCH 02/15] feat(load-test): write load-test reports to output/load-tests//.json Drop the benchmark-specific UI page and report-api route added in the previous commit. Instead the load-test worker writes a normal load-test JSON report to output/load-tests//.json, which the benchmarking uploader can publish alongside regular benchmark runs. Network is resolved from BASE_BENCH_LOAD_TEST_NETWORK env var, then the payload's network field, then a chain-ID fallback map. The path is passed to base-load-test via LOAD_TEST_OUTPUT. The worker now shuts down gracefully so the sequencer can drain remaining transactions before stopping. Also fixes an inverted slices.Contains guard in the importer that would have returned an error for optional (rather than required) missing files. --- report/src/App.tsx | 5 -- report/src/components/RunList.tsx | 21 ++------- report/src/services/dataService.ts | 17 ------- report/src/types.ts | 4 -- report/src/utils/useDataSeries.ts | 26 ----------- runner/benchmark/result_metadata.go | 2 + runner/importer/service.go | 3 +- runner/network/sequencer_benchmark.go | 21 ++++----- runner/network/types/types.go | 4 +- runner/payload/loadtest/load_test_worker.go | 8 +--- runner/service.go | 51 ++++++++++++++++++++- 11 files changed, 71 insertions(+), 91 deletions(-) diff --git a/report/src/App.tsx b/report/src/App.tsx index a1050e76..cef213a2 100644 --- a/report/src/App.tsx +++ b/report/src/App.tsx @@ -5,7 +5,6 @@ import RedirectToLatestRun from "./pages/RedirectToLatestRun"; import LoadTestLanding from "./pages/LoadTestLanding"; import LoadTestAllRuns from "./pages/LoadTestAllRuns"; import LoadTestDetail from "./pages/LoadTestDetail"; -import BenchmarkLoadTestDetail from "./pages/BenchmarkLoadTestDetail"; import ErrorBoundary from "./components/ErrorBoundary"; function App() { @@ -23,10 +22,6 @@ function App() { path="/load-tests/:network/:timestamp" element={} /> - } - /> } /> -
- - {run.result?.artifacts?.loadTestResult && ( - - Load test - - )} -
+ {Object.keys(COLUMN_DEFINITIONS).map((column) => ( { - const response = await fetch( - `${this.baseUrl}output/${encodeURIComponent(outputDir)}/${encodeURIComponent(artifactPath)}`, - ); - - if (!response.ok) { - throw new Error( - `Failed to fetch benchmark load test result: ${response.status} ${response.statusText}`, - ); - } - - return await response.json(); - } } // Configuration helper to determine base URL from environment diff --git a/report/src/types.ts b/report/src/types.ts index ee3f9ffa..13d8c212 100644 --- a/report/src/types.ts +++ b/report/src/types.ts @@ -87,10 +87,6 @@ export interface BenchmarkRun { gasPerSecond: number; newPayload: number; }; - artifacts?: { - loadTestResult?: string; - [key: string]: string | undefined; - }; } | null; } diff --git a/report/src/utils/useDataSeries.ts b/report/src/utils/useDataSeries.ts index 5f36a7b7..24f08eca 100644 --- a/report/src/utils/useDataSeries.ts +++ b/report/src/utils/useDataSeries.ts @@ -139,29 +139,3 @@ export const useLoadTestResult = ( }, ); }; - -export const useBenchmarkLoadTestResult = ( - outputDir: string | undefined, - artifactPath?: string, -) => { - const fetcher = useCallback(async (): Promise => { - if (!outputDir) { - throw new Error("outputDir required"); - } - const dataService = getDataService(); - return await dataService.getBenchmarkLoadTestResult(outputDir, artifactPath); - }, [outputDir, artifactPath]); - - return useSWR( - outputDir - ? `benchmark-load-test-${outputDir}-${artifactPath ?? "load-test-result.json"}` - : null, - fetcher, - { - dedupingInterval: 12 * 60 * 60 * 1000, - revalidateOnFocus: false, - errorRetryCount: 3, - errorRetryInterval: 5000, - }, - ); -}; diff --git a/runner/benchmark/result_metadata.go b/runner/benchmark/result_metadata.go index 641560be..f0e2c5ce 100644 --- a/runner/benchmark/result_metadata.go +++ b/runner/benchmark/result_metadata.go @@ -55,6 +55,8 @@ const ( BenchmarkRunTag = "BenchmarkRun" LoadTestResultArtifactKey = "loadTestResult" LoadTestResultFileName = "load-test-result.json" + LoadTestResultsDir = "load-tests" + LoadTestTimestampLayout = "2006-01-02-15-04-05" ) func RunGroupFromTestPlans(testPlans []TestPlan, machineInfo *MachineInfo) RunGroup { diff --git a/runner/importer/service.go b/runner/importer/service.go index 1ed2008d..e771abb6 100644 --- a/runner/importer/service.go +++ b/runner/importer/service.go @@ -75,7 +75,6 @@ func (s *Service) downloadOutputFiles(baseURL, runID, runOutputDir string) error "result-sequencer.json", "metrics-validator.json", "metrics-sequencer.json", - benchmark.LoadTestResultFileName, } requiredFiles := []string{ @@ -98,7 +97,7 @@ func (s *Service) downloadOutputFiles(baseURL, runID, runOutputDir string) error // Try to download the file err := s.downloadFile(fileURL, localFilePath) if err != nil { - if slices.Contains(requiredFiles, fileName) { + if !slices.Contains(requiredFiles, fileName) { return errors.Wrap(err, "failed to download file") } s.log.Warn("Failed to download file (continuing)", "file", fileName, "url", fileURL, "error", err) diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index 1a7e790d..83e493c4 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -285,9 +285,8 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. payloads = append(payloads, *payload) } - pendingTxs, shutdownErr := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs) - if shutdownErr != nil { - errChan <- shutdownErr + if err := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs); err != nil { + errChan <- err return } @@ -322,14 +321,14 @@ func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( transactionWorker payloadworker.Worker, consensusClient *consensus.SequencerConsensusClient, pendingTxs int, -) (int, error) { +) error { gracefulWorker, ok := transactionWorker.(payloadworker.GracefulShutdownWorker) if !ok { - return pendingTxs, nil + return nil } if err := gracefulWorker.BeginGracefulShutdown(ctx); err != nil { - return pendingTxs, errors.Wrap(err, "failed to begin graceful payload worker shutdown") + return errors.Wrap(err, "failed to begin graceful payload worker shutdown") } timeout := time.NewTimer(gracefulWorkerShutdownTimeout) @@ -340,25 +339,25 @@ func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( select { case <-gracefulWorker.Done(): nb.log.Info("Payload worker stopped gracefully", "settlement_blocks", settlementBlock) - return pendingTxs, nil + return nil case <-timeout.C: nb.log.Warn("Timed out waiting for payload worker to stop gracefully", "settlement_blocks", settlementBlock) - return pendingTxs, nil + return nil default: } blockMetrics := metrics.NewBlockMetrics() txsSent, err := transactionWorker.SendTxs(ctx, pendingTxs) if err != nil { - return pendingTxs, errors.Wrap(err, "failed to collect settlement transactions") + return errors.Wrap(err, "failed to collect settlement transactions") } payload, err := consensusClient.Propose(ctx, blockMetrics, true) if err != nil { - return pendingTxs, errors.Wrap(err, "failed to propose settlement block") + return errors.Wrap(err, "failed to propose settlement block") } if payload == nil { - return pendingTxs, errors.New("received nil settlement payload from consensus client") + return errors.New("received nil settlement payload from consensus client") } userTxsIncluded := len(payload.Transactions) - 1 diff --git a/runner/network/types/types.go b/runner/network/types/types.go index 2b93c1e5..9bb0b34a 100644 --- a/runner/network/types/types.go +++ b/runner/network/types/types.go @@ -41,8 +41,8 @@ type TestConfig struct { PrefundPrivateKey ecdsa.PrivateKey PrefundAmount big.Int - // LoadTestOutputPath is the optional JSON summary path used by the - // load-test payload worker. + // LoadTestOutputPath is the optional normal load-test report JSON path used + // by the load-test payload worker. LoadTestOutputPath string } diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index 0723496c..a236924d 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -30,6 +30,7 @@ import ( type LoadTestPayloadDefinition struct { SenderCount uint64 `yaml:"sender_count"` FundingAmount string `yaml:"funding_amount"` + Network string `yaml:"network"` Transactions yaml.Node `yaml:"transactions"` } @@ -60,8 +61,6 @@ type loadTestPayloadWorker struct { proxyServer *proxy.ProxyServer cmd *exec.Cmd done chan struct{} - waitErr error - waitMu sync.Mutex shutdownOnce sync.Once configFilePath string outputPath string @@ -140,10 +139,7 @@ func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { w.cmd = cmd w.done = make(chan struct{}) go func() { - err := cmd.Wait() - w.waitMu.Lock() - w.waitErr = err - w.waitMu.Unlock() + _ = cmd.Wait() close(w.done) }() diff --git a/runner/service.go b/runner/service.go index 2add585f..2d9e1e31 100644 --- a/runner/service.go +++ b/runner/service.go @@ -26,6 +26,7 @@ import ( "github.com/base/base-bench/runner/network" "github.com/base/base-bench/runner/network/types" "github.com/base/base-bench/runner/payload" + "github.com/base/base-bench/runner/payload/loadtest" "github.com/base/base-bench/runner/utils" "github.com/ethereum/go-ethereum/core" ethparams "github.com/ethereum/go-ethereum/params" @@ -373,6 +374,54 @@ func (s *service) setupBlobsDir(workingDir string) error { return nil } +func loadTestNetwork(genesis *core.Genesis, transactionPayload payload.Definition) string { + if envNetwork := os.Getenv("BASE_BENCH_LOAD_TEST_NETWORK"); envNetwork != "" { + return envNetwork + } + + if transactionPayload.Type == "load-test" { + if def, ok := transactionPayload.Params.(*loadtest.LoadTestPayloadDefinition); ok && def.Network != "" { + return def.Network + } + } + + if genesis == nil || genesis.Config == nil || genesis.Config.ChainID == nil { + return "unknown" + } + + switch genesis.Config.ChainID.Uint64() { + case 8453: + return "mainnet" + case 84532: + return "sepolia" + case 13371337: + return "devnet" + default: + return fmt.Sprintf("chain-%s", genesis.Config.ChainID.String()) + } +} + +func (s *service) loadTestOutputPath(genesis *core.Genesis, transactionPayload payload.Definition) string { + if transactionPayload.Type != "load-test" { + return "" + } + + network := loadTestNetwork(genesis, transactionPayload) + baseTime := time.Now().UTC() + for i := 0; ; i++ { + timestamp := baseTime.Add(time.Duration(i) * time.Second).Format(benchmark.LoadTestTimestampLayout) + outputPath := path.Join( + s.config.OutputDir(), + benchmark.LoadTestResultsDir, + network, + fmt.Sprintf("%s.json", timestamp), + ) + if _, err := os.Stat(outputPath); err != nil { + return outputPath + } + } +} + func (s *service) runTest(ctx context.Context, params types.RunParams, workingDir string, outputDir string, snapshotConfig *benchmark.SnapshotDefinition, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition, datadirsConfig *benchmark.DatadirConfig, mode benchmark.BenchmarkExecutionMode, flashblocksBlockTime string) (*benchmark.RunResult, error) { s.log.Info(fmt.Sprintf("Running benchmark with params: %+v", params)) @@ -434,7 +483,7 @@ func (s *service) runTest(ctx context.Context, params types.RunParams, workingDi BatcherKey: *batcherKey, PrefundPrivateKey: *prefundKey, PrefundAmount: *prefundAmount, - LoadTestOutputPath: path.Join(outputDir, benchmark.LoadTestResultFileName), + LoadTestOutputPath: s.loadTestOutputPath(genesis, transactionPayload), } // Run benchmark From e40909442839de9f6b67f5c4f1afa7bd1a6066d1 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Mon, 18 May 2026 16:21:29 -0500 Subject: [PATCH 03/15] feat(load-test): use native config files --- configs/examples/load-test-config.yml | 20 ++ configs/examples/load-test.yml | 12 +- runner/payload/loadtest/load_test_worker.go | 230 ++++++++++-------- .../payload/loadtest/load_test_worker_test.go | 125 ++++++++++ 4 files changed, 269 insertions(+), 118 deletions(-) create mode 100644 configs/examples/load-test-config.yml create mode 100644 runner/payload/loadtest/load_test_worker_test.go diff --git a/configs/examples/load-test-config.yml b/configs/examples/load-test-config.yml new file mode 100644 index 00000000..77074cdd --- /dev/null +++ b/configs/examples/load-test-config.yml @@ -0,0 +1,20 @@ +transaction_submission_rpcs: + - "http://localhost:8545" +query_rpc: "http://localhost:8545" +txpool_nodes: [] +flashblocks_ws: "ws://localhost:7111" + +sender_count: 10 +target_gps: 500000000 +duration: "60s" +funding_amount: "10000000000000000000" + +transactions: + - weight: 70 + type: transfer + - weight: 20 + type: calldata + max_size: 256 + - weight: 10 + type: precompile + target: sha256 diff --git a/configs/examples/load-test.yml b/configs/examples/load-test.yml index 6743ca60..163513ce 100644 --- a/configs/examples/load-test.yml +++ b/configs/examples/load-test.yml @@ -4,16 +4,8 @@ payloads: - name: Load Test type: load-test id: load-test - sender_count: 10 - transactions: - - weight: 70 - type: transfer - - weight: 20 - type: calldata - max_size: 256 - - weight: 10 - type: precompile - target: sha256 + network: devnet + config_file: ./load-test-config.yml benchmarks: - variables: diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index a236924d..92527aa8 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -3,13 +3,13 @@ package loadtest import ( "context" "crypto/ecdsa" - cryptorand "crypto/rand" "encoding/hex" "fmt" "math/big" "os" "os/exec" "path/filepath" + "strconv" "sync" "time" @@ -24,46 +24,30 @@ import ( ) // LoadTestPayloadDefinition is the YAML payload params for the load-test type. -// Fields map directly to the Rust base-load-test config format. -// The `transactions` field is passed through as raw YAML to support the full -// Rust config schema (transfer, calldata, precompile, erc20, etc.). +// The load-test workload itself lives in a native base-load-tester config file; +// benchmark mode only overlays the RPC and timing fields it must control. type LoadTestPayloadDefinition struct { - SenderCount uint64 `yaml:"sender_count"` - FundingAmount string `yaml:"funding_amount"` - Network string `yaml:"network"` - Transactions yaml.Node `yaml:"transactions"` -} - -// loadTestConfig is the YAML config written to a temp file for the load-test binary. -type loadTestConfig struct { - RPC string `yaml:"rpc,omitempty"` - TransactionSubmissionRPCs []string `yaml:"transaction_submission_rpcs"` - QueryRPC string `yaml:"query_rpc"` - FlashblocksWs string `yaml:"flashblocks_ws"` - SenderCount uint64 `yaml:"sender_count"` - TargetGPS uint64 `yaml:"target_gps"` - Duration string `yaml:"duration"` - Seed uint64 `yaml:"seed"` - FundingAmount string `yaml:"funding_amount"` - Transactions yaml.Node `yaml:"transactions"` + ConfigFile string `yaml:"config_file"` + Network string `yaml:"network"` } type loadTestPayloadWorker struct { - log log.Logger - prefundSK string - loadTestBin string - elRPCURL string - flashblocksURL string - gasLimit uint64 - blockTimeSec uint64 - params LoadTestPayloadDefinition - mempool *mempool.StaticWorkloadMempool - proxyServer *proxy.ProxyServer - cmd *exec.Cmd - done chan struct{} - shutdownOnce sync.Once - configFilePath string - outputPath string + log log.Logger + prefundSK string + loadTestBin string + elRPCURL string + flashblocksURL string + gasLimit uint64 + blockTimeSec uint64 + params LoadTestPayloadDefinition + mempool *mempool.StaticWorkloadMempool + proxyServer *proxy.ProxyServer + cmd *exec.Cmd + done chan struct{} + shutdownOnce sync.Once + sourceConfigPath string + renderedConfigPath string + outputPath string } // NewLoadTestPayloadWorker creates a worker that runs the base-load-test binary @@ -88,18 +72,24 @@ func NewLoadTestPayloadWorker( blockTimeSec = 1 } + sourceConfigPath, err := resolveConfigFilePath(cfg.ConfigPath(), definition.ConfigFile) + if err != nil { + return nil, err + } + w := &loadTestPayloadWorker{ - log: log, - prefundSK: hex.EncodeToString(prefundedPrivateKey.D.Bytes()), - loadTestBin: cfg.LoadTestBinary(), - elRPCURL: elRPCURL, - flashblocksURL: flashblocksURL, - gasLimit: params.GasLimit, - blockTimeSec: blockTimeSec, - params: definition, - mempool: mp, - proxyServer: ps, - outputPath: outputPath, + log: log, + prefundSK: hex.EncodeToString(prefundedPrivateKey.D.Bytes()), + loadTestBin: cfg.LoadTestBinary(), + elRPCURL: elRPCURL, + flashblocksURL: flashblocksURL, + gasLimit: params.GasLimit, + blockTimeSec: blockTimeSec, + params: definition, + mempool: mp, + proxyServer: ps, + sourceConfigPath: sourceConfigPath, + outputPath: outputPath, } return w, nil @@ -118,7 +108,7 @@ func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to write load-test config") } - w.configFilePath = configPath + w.renderedConfigPath = configPath w.log.Info("Starting load test", "binary", w.loadTestBin, "config", configPath) @@ -207,9 +197,9 @@ func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { w.proxyServer.Stop() - if w.configFilePath != "" { - if err := os.Remove(w.configFilePath); err != nil { - w.log.Warn("failed to remove load-test config", "path", w.configFilePath, "err", err) + if w.renderedConfigPath != "" { + if err := os.Remove(w.renderedConfigPath); err != nil { + w.log.Warn("failed to remove load-test config", "path", w.renderedConfigPath, "err", err) } } @@ -224,80 +214,104 @@ func (w *loadTestPayloadWorker) SendTxs(ctx context.Context, _ int) (int, error) return len(pendingTxs), nil } -// defaultTransactions returns the default transaction mix as a yaml.Node. -func defaultTransactions() yaml.Node { - var node yaml.Node - // Default: 70% transfer, 20% calldata, 10% precompile - defaultYAML := ` -- weight: 70 - type: transfer -- weight: 20 - type: calldata - max_size: 256 -- weight: 10 - type: precompile - target: sha256 -` - if err := yaml.Unmarshal([]byte(defaultYAML), &node); err != nil { - panic(fmt.Sprintf("failed to parse default transactions YAML: %v", err)) +func resolveConfigFilePath(benchmarkConfigPath string, loadTestConfigPath string) (string, error) { + if loadTestConfigPath == "" { + return "", errors.New("load-test payload requires config_file") } - // yaml.Unmarshal wraps in a document node; return the inner sequence - if node.Kind == yaml.DocumentNode && len(node.Content) > 0 { - return *node.Content[0] + if filepath.IsAbs(loadTestConfigPath) { + return loadTestConfigPath, nil } - return node + return filepath.Join(filepath.Dir(benchmarkConfigPath), loadTestConfigPath), nil } -// randomSeed returns a cryptographically random uint64 seed. -func randomSeed() uint64 { - var b [8]byte - if _, err := cryptorand.Read(b[:]); err != nil { - return 42 +func (w *loadTestPayloadWorker) targetGPS() uint64 { + blockTimeSec := w.blockTimeSec + if blockTimeSec == 0 { + blockTimeSec = 1 } - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return w.gasLimit / blockTimeSec } -// writeConfig generates a temporary YAML config file for the load-test binary -// with the RPC URL pointing to the proxy server. -func (w *loadTestPayloadWorker) writeConfig() (string, error) { - senderCount := w.params.SenderCount - if senderCount == 0 { - senderCount = 10 +func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { + data, err := os.ReadFile(w.sourceConfigPath) + if err != nil { + return nil, errors.Wrap(err, "failed to read load-test config file") } - fundingAmount := w.params.FundingAmount - if fundingAmount == "" { - fundingAmount = "10000000000000000000" + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, errors.Wrap(err, "failed to parse load-test config file") } - // Compute target GPS from gas limit and block time - targetGPS := w.gasLimit / w.blockTimeSec - - transactions := w.params.Transactions - if transactions.Kind == 0 { - transactions = defaultTransactions() + config, err := mappingRoot(&doc) + if err != nil { + return nil, err } + proxyURL := w.proxyServer.ClientURL() + setMappingValue(config, "transaction_submission_rpcs", stringSequenceNode(proxyURL)) + setMappingValue(config, "query_rpc", stringNode(proxyURL)) + flashblocksURL := w.flashblocksURL if flashblocksURL == "" { flashblocksURL = "ws://localhost:7111" } + setMappingValue(config, "flashblocks_ws", stringNode(flashblocksURL)) + setMappingValue(config, "target_gps", uintNode(w.targetGPS())) + setMappingValue(config, "duration", stringNode("99999s")) + + return config, nil +} + +func mappingRoot(doc *yaml.Node) (*yaml.Node, error) { + root := doc + if doc.Kind == yaml.DocumentNode { + if len(doc.Content) == 0 { + return nil, errors.New("load-test config file is empty") + } + root = doc.Content[0] + } - config := loadTestConfig{ - RPC: w.proxyServer.ClientURL(), - TransactionSubmissionRPCs: []string{w.proxyServer.ClientURL()}, - QueryRPC: w.proxyServer.ClientURL(), - FlashblocksWs: flashblocksURL, - SenderCount: senderCount, - TargetGPS: targetGPS, - Duration: "99999s", - Seed: randomSeed(), - FundingAmount: fundingAmount, - Transactions: transactions, + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("load-test config file must be a YAML mapping, got kind %d", root.Kind) + } + return root, nil +} + +func setMappingValue(mapping *yaml.Node, key string, value *yaml.Node) { + for i := 0; i < len(mapping.Content)-1; i += 2 { + if mapping.Content[i].Value == key { + mapping.Content[i+1] = value + return + } } + mapping.Content = append(mapping.Content, stringNode(key), value) +} + +func stringNode(value string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value} +} + +func uintNode(value uint64) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: strconv.FormatUint(value, 10)} +} - data, err := yaml.Marshal(&config) +func stringSequenceNode(values ...string) *yaml.Node { + node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for _, value := range values { + node.Content = append(node.Content, stringNode(value)) + } + return node +} + +// writeConfig generates a temporary YAML config file for the load-test binary +// with the RPC URL pointing to the proxy server. +func (w *loadTestPayloadWorker) writeConfig() (string, error) { + config, err := w.buildConfig() + if err != nil { + return "", err + } + data, err := yaml.Marshal(config) if err != nil { return "", errors.Wrap(err, "failed to marshal load-test config") } @@ -317,8 +331,8 @@ func (w *loadTestPayloadWorker) writeConfig() (string, error) { } w.log.Info("Generated load-test config", - "sender_count", senderCount, - "target_gps", targetGPS, + "source_config", w.sourceConfigPath, + "target_gps", w.targetGPS(), "gas_limit", w.gasLimit, "block_time_sec", w.blockTimeSec, ) diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go new file mode 100644 index 00000000..b69e8784 --- /dev/null +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -0,0 +1,125 @@ +package loadtest + +import ( + "os" + "path/filepath" + "testing" + + "github.com/base/base-bench/runner/clients/common/proxy" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestBuildConfigOverlaysBenchmarkFieldsAndPreservesLoadTestConfig(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "mainnet-state-weth-usdc-swaps.yaml") + err := os.WriteFile(configPath, []byte(` +transaction_submission_rpcs: + - "http://standalone-submitter.invalid" +query_rpc: "http://standalone-query.invalid" +flashblocks_ws: "ws://standalone-flashblocks.invalid" +target_gps: 123 +duration: "60s" +chain_id: 8453 +sender_count: 250 +in_flight_per_sender: 64 +batch_size: 20 +batch_timeout: "10ms" +seed: 654789 +funding_amount: "200000000000000000" +real_token_setup: + enabled: true + allow_chain_id_8453: true + weth: "0x4200000000000000000000000000000000000006" + weth_amount_per_sender: "50000000000000000" + pair_token: + token: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913" + amount_per_sender: "10000000" + acquisition: + type: uniswap_v3_exact_input + router: "0x2626664c2603336E57B271c5C0b26F421741e481" + fee: 500 + amount_in: "10000000000000000" + min_amount_out: "0" +transactions: + - weight: 50 + type: uniswap_v3 + router: "0x2626664c2603336E57B271c5C0b26F421741e481" + token_in: "0x4200000000000000000000000000000000000006" + token_out: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913" + fee: 500 + min_amount: "10000000000000" + max_amount: "100000000000000" + reverse_min_amount: "100000" + reverse_max_amount: "1000000" + - weight: 50 + type: aerodrome_cl + router: "0xBE6D8f0d05cC4be24d5167a3eF062215bE6D18a5" + token_in: "0x4200000000000000000000000000000000000006" + token_out: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913" + tick_spacing: 100 + min_amount: "10000000000000" + max_amount: "100000000000000" + reverse_min_amount: "100000" + reverse_max_amount: "1000000" +`), 0644) + require.NoError(t, err) + + worker := &loadTestPayloadWorker{ + flashblocksURL: "ws://benchmark-flashblocks.example", + gasLimit: 150_000_000, + blockTimeSec: 2, + proxyServer: proxy.NewProxyServer("http://sequencer.example", nil, 18546, nil), + sourceConfigPath: configPath, + } + + config, err := worker.buildConfig() + require.NoError(t, err) + + encoded, err := yaml.Marshal(config) + require.NoError(t, err) + output := string(encoded) + + for _, want := range []string{ + "transaction_submission_rpcs:\n - http://localhost:18546", + "query_rpc: http://localhost:18546", + "flashblocks_ws: ws://benchmark-flashblocks.example", + "target_gps: 75000000", + "duration: 99999s", + "chain_id: 8453", + "sender_count: 250", + "in_flight_per_sender: 64", + "batch_size: 20", + "batch_timeout: \"10ms\"", + "seed: 654789", + "real_token_setup:", + "allow_chain_id_8453: true", + "type: uniswap_v3", + "type: aerodrome_cl", + "reverse_min_amount: \"100000\"", + } { + require.Contains(t, output, want) + } + for _, oldValue := range []string{ + "standalone-submitter.invalid", + "standalone-query.invalid", + "standalone-flashblocks.invalid", + "target_gps: 123", + "duration: 60s", + } { + require.NotContains(t, output, oldValue) + } +} + +func TestResolveConfigFilePath(t *testing.T) { + resolved, err := resolveConfigFilePath("/tmp/configs/benchmark.yml", "load-tests/mainnet.yaml") + require.NoError(t, err) + require.Equal(t, "/tmp/configs/load-tests/mainnet.yaml", resolved) + + resolved, err = resolveConfigFilePath("/tmp/configs/benchmark.yml", "/var/load-tests/mainnet.yaml") + require.NoError(t, err) + require.Equal(t, "/var/load-tests/mainnet.yaml", resolved) + + _, err = resolveConfigFilePath("/tmp/configs/benchmark.yml", "") + require.Error(t, err) + require.Contains(t, err.Error(), "config_file") +} From d34d5e47eae61bb33f69dd7169582b72b877b55e Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Tue, 19 May 2026 15:19:11 -0500 Subject: [PATCH 04/15] fix(load-test): proxy snapshot nonces from upstream Forward pass-through JSON-RPC batch items to the upstream node while continuing to capture raw transactions. Fetch eth_getTransactionCount from upstream and merge it with transactions already accepted by the proxy so snapshot-backed accounts keep their real chain nonce. Co-authored-by: Codex --- runner/clients/common/proxy/proxy.go | 199 +++++++++++++--- runner/clients/common/proxy/proxy_test.go | 266 +++++++++++++++++++++- 2 files changed, 433 insertions(+), 32 deletions(-) diff --git a/runner/clients/common/proxy/proxy.go b/runner/clients/common/proxy/proxy.go index b212ec0b..bf3f8147 100644 --- a/runner/clients/common/proxy/proxy.go +++ b/runner/clients/common/proxy/proxy.go @@ -21,6 +21,7 @@ import ( "github.com/base/base-bench/runner/network/mempool" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" ethTypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" ) @@ -32,15 +33,31 @@ type ProxyServer struct { pendingTxs []*ethTypes.Transaction clientURL string mempool *mempool.StaticWorkloadMempool + nextNonce map[common.Address]uint64 mu sync.Mutex } +type rpcRequest struct { + Method string `json:"method"` + Params json.RawMessage `json:"params"` + ID interface{} `json:"id"` + JSONRPC string `json:"jsonrpc"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error interface{} `json:"error,omitempty"` +} + func NewProxyServer(clientURL string, log log.Logger, port int, mempool *mempool.StaticWorkloadMempool) *ProxyServer { return &ProxyServer{ clientURL: clientURL, log: log, port: port, mempool: mempool, + nextNonce: make(map[common.Address]uint64), } } @@ -109,13 +126,6 @@ func (p *ProxyServer) handleRequest(w http.ResponseWriter, r *http.Request) { return } - type rpcRequest struct { - Method string `json:"method"` - Params json.RawMessage `json:"params"` - ID interface{} `json:"id"` - JSONRPC string `json:"jsonrpc"` - } - if len(body) > 0 && body[0] == '[' { p.handleBatchRequest(w, body) return @@ -135,11 +145,7 @@ func (p *ProxyServer) handleRequest(w http.ResponseWriter, r *http.Request) { } if handled { - resp := struct { - JSONRPC string `json:"jsonrpc"` - ID interface{} `json:"id"` - Result json.RawMessage `json:"result"` - }{ + resp := rpcResponse{ JSONRPC: request.JSONRPC, ID: request.ID, Result: response, @@ -192,26 +198,12 @@ func (p *ProxyServer) handleRequest(w http.ResponseWriter, r *http.Request) { } func (p *ProxyServer) handleBatchRequest(w http.ResponseWriter, body []byte) { - type rpcRequest struct { - Method string `json:"method"` - Params json.RawMessage `json:"params"` - ID interface{} `json:"id"` - JSONRPC string `json:"jsonrpc"` - } - var requests []rpcRequest if err := json.Unmarshal(body, &requests); err != nil { http.Error(w, "Error parsing batch request", http.StatusBadRequest) return } - type rpcResponse struct { - JSONRPC string `json:"jsonrpc"` - ID interface{} `json:"id"` - Result json.RawMessage `json:"result,omitempty"` - Error interface{} `json:"error,omitempty"` - } - responses := make([]rpcResponse, 0, len(requests)) for _, request := range requests { handled, result, err := p.OverrideRequest(request.Method, request.Params) @@ -224,7 +216,12 @@ func (p *ProxyServer) handleBatchRequest(w http.ResponseWriter, body []byte) { } else if handled { response.Result = result } else { - response.Error = map[string]interface{}{"code": -32601, "message": "method not supported in proxy batch mode"} + forwardedResponse, err := p.forwardRPCRequest(request) + if err != nil { + response.Error = map[string]interface{}{"code": -32000, "message": err.Error()} + } else { + response = forwardedResponse + } } responses = append(responses, response) } @@ -235,6 +232,43 @@ func (p *ProxyServer) handleBatchRequest(w http.ResponseWriter, body []byte) { } } +func (p *ProxyServer) forwardRPCRequest(request rpcRequest) (rpcResponse, error) { + body, err := json.Marshal(request) + if err != nil { + return rpcResponse{}, fmt.Errorf("failed to marshal upstream request: %w", err) + } + + resp, err := http.Post(p.clientURL, "application/json", bytes.NewReader(body)) + if err != nil { + return rpcResponse{}, fmt.Errorf("failed to forward request: %w", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + p.log.Error("Error closing response body", "err", err) + } + }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return rpcResponse{}, fmt.Errorf("failed to read upstream response: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return rpcResponse{}, fmt.Errorf("upstream request returned status %d: %s", resp.StatusCode, string(respBody)) + } + + var forwardedResponse rpcResponse + if err := json.Unmarshal(respBody, &forwardedResponse); err != nil { + return rpcResponse{}, fmt.Errorf("failed to decode upstream response: %w", err) + } + if forwardedResponse.JSONRPC == "" { + forwardedResponse.JSONRPC = request.JSONRPC + } + if forwardedResponse.ID == nil { + forwardedResponse.ID = request.ID + } + return forwardedResponse, nil +} + func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) (bool, json.RawMessage, error) { switch method { case "eth_getTransactionCount": @@ -242,8 +276,22 @@ func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) if err := json.Unmarshal(rawParams, ¶ms); err != nil { return false, nil, fmt.Errorf("failed to unmarshal params: %w", err) } + if len(params) == 0 { + return false, nil, fmt.Errorf("no params found") + } - nonce := p.mempool.GetTransactionCount(common.HexToAddress(params[0])) + address := common.HexToAddress(params[0]) + nonce, err := p.upstreamTransactionCount(rawParams) + if err != nil { + if observedNonce, ok := p.observedTransactionCount(address); ok { + jsonResponse, _ := json.Marshal(fmt.Sprintf("0x%x", observedNonce)) + return true, jsonResponse, nil + } + return false, nil, err + } + if observedNonce, ok := p.observedTransactionCount(address); ok && observedNonce > nonce { + nonce = observedNonce + } jsonResponse, _ := json.Marshal(fmt.Sprintf("0x%x", nonce)) return true, jsonResponse, nil @@ -275,9 +323,7 @@ func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) return false, nil, fmt.Errorf("failed to decode transaction: %w", err) } - p.mu.Lock() - p.pendingTxs = append(p.pendingTxs, &tx) - p.mu.Unlock() + p.recordPendingTransaction(&tx) txHash := tx.Hash().Hex() jsonResponse, _ := json.Marshal(txHash) @@ -287,10 +333,101 @@ func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) } } +func (p *ProxyServer) upstreamTransactionCount(rawParams json.RawMessage) (uint64, error) { + body, err := json.Marshal(struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + }{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_getTransactionCount", + Params: rawParams, + }) + if err != nil { + return 0, fmt.Errorf("failed to marshal upstream nonce request: %w", err) + } + + resp, err := http.Post(p.clientURL, "application/json", bytes.NewReader(body)) + if err != nil { + return 0, fmt.Errorf("failed to fetch upstream transaction count: %w", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + p.log.Error("Error closing response body", "err", err) + } + }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return 0, fmt.Errorf("failed to read upstream transaction count response: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return 0, fmt.Errorf("upstream transaction count request returned status %d: %s", resp.StatusCode, string(respBody)) + } + + var rpcResp struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(respBody, &rpcResp); err != nil { + return 0, fmt.Errorf("failed to decode upstream transaction count response: %w", err) + } + if rpcResp.Error != nil { + return 0, fmt.Errorf("upstream transaction count error %d: %s", rpcResp.Error.Code, rpcResp.Error.Message) + } + + var nonceHex string + if err := json.Unmarshal(rpcResp.Result, &nonceHex); err != nil { + return 0, fmt.Errorf("failed to decode upstream transaction count result: %w", err) + } + nonce, err := hexutil.DecodeUint64(nonceHex) + if err != nil { + return 0, fmt.Errorf("failed to parse upstream transaction count %q: %w", nonceHex, err) + } + return nonce, nil +} + +func (p *ProxyServer) observedTransactionCount(address common.Address) (uint64, bool) { + p.mu.Lock() + defer p.mu.Unlock() + + nonce, ok := p.nextNonce[address] + return nonce, ok +} + +func (p *ProxyServer) recordPendingTransaction(tx *ethTypes.Transaction) { + from, err := ethTypes.Sender(ethTypes.LatestSignerForChainID(tx.ChainId()), tx) + if err != nil { + p.log.Warn("failed to recover sender for observed transaction", "err", err, "hash", tx.Hash()) + p.mu.Lock() + p.pendingTxs = append(p.pendingTxs, tx) + p.mu.Unlock() + return + } + + nextNonce := tx.Nonce() + 1 + p.mu.Lock() + p.pendingTxs = append(p.pendingTxs, tx) + if nextNonce > p.nextNonce[from] { + p.nextNonce[from] = nextNonce + } + p.mu.Unlock() +} + func (p *ProxyServer) DebugResponse(method string, params json.RawMessage, respBody []byte) { p.log.Debug("method", "method", method) p.log.Debug("params", "params", params) + if !bytes.HasPrefix(respBody, []byte{0x1f, 0x8b}) { + p.log.Debug("Response body", "body", string(respBody)) + return + } + gzipReader, err := gzip.NewReader(bytes.NewReader(respBody)) if err != nil { p.log.Error("Error creating gzip reader", "err", err) diff --git a/runner/clients/common/proxy/proxy_test.go b/runner/clients/common/proxy/proxy_test.go index a5c0678f..013f3ebe 100644 --- a/runner/clients/common/proxy/proxy_test.go +++ b/runner/clients/common/proxy/proxy_test.go @@ -78,7 +78,271 @@ func TestHandleBatchRequestCapturesRawTransactions(t *testing.T) { } } +func TestHandleBatchRequestForwardsPassThroughMethods(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req rpcRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode upstream request: %v", err) + } + if req.Method != "eth_chainId" { + t.Fatalf("expected eth_chainId, got %s", req.Method) + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "jsonrpc": "2.0", + "id": req.ID, + "result": "0x2105", + }); err != nil { + t.Fatalf("encode upstream response: %v", err) + } + })) + defer upstream.Close() + + server := NewProxyServer( + upstream.URL, + log.New(), + 0, + mempool.NewStaticWorkloadMempool(log.New(), big.NewInt(8453)), + ) + + body, err := json.Marshal([]map[string]any{ + { + "jsonrpc": "2.0", + "id": 7, + "method": "eth_chainId", + "params": []string{}, + }, + }) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + rec := httptest.NewRecorder() + + server.handleRequest(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", rec.Code, rec.Body.String()) + } + + var responses []struct { + Result string `json:"result"` + Error map[string]any `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &responses); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(responses) != 1 { + t.Fatalf("expected 1 response, got %d", len(responses)) + } + if responses[0].Error != nil { + t.Fatalf("expected successful response, got error %v", responses[0].Error) + } + if responses[0].Result != "0x2105" { + t.Fatalf("expected forwarded result 0x2105, got %s", responses[0].Result) + } +} + +func TestHandleBatchRequestSupportsMixedForwardAndCapture(t *testing.T) { + chainID := big.NewInt(8453) + tx := signedTestTx(t, chainID) + rawTx, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal tx: %v", err) + } + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req rpcRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode upstream request: %v", err) + } + if req.Method != "eth_gasPrice" { + t.Fatalf("expected eth_gasPrice, got %s", req.Method) + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "jsonrpc": "2.0", + "id": req.ID, + "result": "0x3b9aca00", + }); err != nil { + t.Fatalf("encode upstream response: %v", err) + } + })) + defer upstream.Close() + + server := NewProxyServer( + upstream.URL, + log.New(), + 0, + mempool.NewStaticWorkloadMempool(log.New(), chainID), + ) + + body, err := json.Marshal([]map[string]any{ + { + "jsonrpc": "2.0", + "id": 0, + "method": "eth_gasPrice", + "params": []string{}, + }, + { + "jsonrpc": "2.0", + "id": 1, + "method": "eth_sendRawTransaction", + "params": []string{hexutil.Encode(rawTx)}, + }, + }) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + rec := httptest.NewRecorder() + + server.handleRequest(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", rec.Code, rec.Body.String()) + } + + var responses []struct { + Result string `json:"result"` + Error map[string]any `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &responses); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(responses) != 2 { + t.Fatalf("expected 2 responses, got %d", len(responses)) + } + if responses[0].Error != nil { + t.Fatalf("expected successful forwarded response, got error %v", responses[0].Error) + } + if responses[0].Result != "0x3b9aca00" { + t.Fatalf("expected forwarded gas price, got %s", responses[0].Result) + } + if responses[1].Error != nil { + t.Fatalf("expected successful captured tx response, got error %v", responses[1].Error) + } + if responses[1].Result != tx.Hash().Hex() { + t.Fatalf("expected tx hash %s, got %s", tx.Hash().Hex(), responses[1].Result) + } + + pending := server.DrainPendingTxs() + if len(pending) != 1 { + t.Fatalf("expected 1 pending tx, got %d", len(pending)) + } + if pending[0].Hash() != tx.Hash() { + t.Fatalf("expected pending tx %s, got %s", tx.Hash(), pending[0].Hash()) + } +} + +func TestGetTransactionCountForwardsUpstreamNonce(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "result": "0xfa", + }); err != nil { + t.Fatalf("encode upstream response: %v", err) + } + })) + defer upstream.Close() + + server := NewProxyServer( + upstream.URL, + log.New(), + 0, + mempool.NewStaticWorkloadMempool(log.New(), big.NewInt(8453)), + ) + + result := callProxyRPC(t, server, "eth_getTransactionCount", []string{common.Address{1}.Hex(), "pending"}) + if result != "0xfa" { + t.Fatalf("expected upstream nonce 0xfa, got %s", result) + } +} + +func TestGetTransactionCountIncludesObservedPendingTransactions(t *testing.T) { + chainID := big.NewInt(8453) + tx := signedTestTxWithNonce(t, chainID, 250) + from, err := types.Sender(types.LatestSignerForChainID(chainID), tx) + if err != nil { + t.Fatalf("recover sender: %v", err) + } + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "result": "0xfa", + }); err != nil { + t.Fatalf("encode upstream response: %v", err) + } + })) + defer upstream.Close() + + server := NewProxyServer( + upstream.URL, + log.New(), + 0, + mempool.NewStaticWorkloadMempool(log.New(), chainID), + ) + + rawTx, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal tx: %v", err) + } + callProxyRPC(t, server, "eth_sendRawTransaction", []string{hexutil.Encode(rawTx)}) + + result := callProxyRPC(t, server, "eth_getTransactionCount", []string{from.Hex(), "pending"}) + if result != "0xfb" { + t.Fatalf("expected observed nonce 0xfb, got %s", result) + } +} + +func callProxyRPC(t *testing.T, server *ProxyServer, method string, params any) string { + t.Helper() + + body, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": method, + "params": params, + }) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + rec := httptest.NewRecorder() + server.handleRequest(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", rec.Code, rec.Body.String()) + } + + var response struct { + Result string `json:"result"` + Error map[string]any `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if response.Error != nil { + t.Fatalf("expected successful response, got error %v", response.Error) + } + return response.Result +} + func signedTestTx(t *testing.T, chainID *big.Int) *types.Transaction { + return signedTestTxWithNonce(t, chainID, 0) +} + +func signedTestTxWithNonce(t *testing.T, chainID *big.Int, nonce uint64) *types.Transaction { t.Helper() key, err := crypto.GenerateKey() @@ -88,7 +352,7 @@ func signedTestTx(t *testing.T, chainID *big.Int) *types.Transaction { tx := types.NewTx(&types.DynamicFeeTx{ ChainID: chainID, - Nonce: 0, + Nonce: nonce, GasTipCap: big.NewInt(1), GasFeeCap: big.NewInt(1), Gas: 21_000, From 3e21fb21c62cc3bb88a5fa7d0544c4ea50d11ec8 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 20 May 2026 14:30:30 -0500 Subject: [PATCH 05/15] Add target GPS support for load-test benchmarks --- configs/README.md | 2 +- configs/examples/load-test.yml | 2 + runner/benchmark/benchmark.go | 6 ++ runner/benchmark/matrix_test.go | 67 +++++++++++++++++++ runner/network/types/types.go | 7 ++ runner/payload/loadtest/load_test_worker.go | 30 +++------ .../payload/loadtest/load_test_worker_test.go | 35 +++++++++- 7 files changed, 124 insertions(+), 25 deletions(-) diff --git a/configs/README.md b/configs/README.md index 46c17f4c..0825df30 100644 --- a/configs/README.md +++ b/configs/README.md @@ -77,7 +77,7 @@ benchmarks: - name: "Benchmark Name" description: "What this benchmark tests" variables: - - type: payload|node_type|num_blocks|gas_limit + - type: payload|node_type|num_blocks|gas_limit|target_gps value: single-value values: [array, of, values] # for matrix testing ``` diff --git a/configs/examples/load-test.yml b/configs/examples/load-test.yml index 163513ce..143b9774 100644 --- a/configs/examples/load-test.yml +++ b/configs/examples/load-test.yml @@ -19,3 +19,5 @@ benchmarks: value: 10 - type: gas_limit value: 1000000000 + - type: target_gps + value: 500000000 diff --git a/runner/benchmark/benchmark.go b/runner/benchmark/benchmark.go index f1614af3..cd82985a 100644 --- a/runner/benchmark/benchmark.go +++ b/runner/benchmark/benchmark.go @@ -70,6 +70,12 @@ func NewParamsFromValues(assignments map[string]interface{}) (*types.RunParams, } else { return nil, fmt.Errorf("invalid gas limit %s", v) } + case "target_gps": + if vInt, ok := v.(int); ok { + params.TargetGPS = uint64(vInt) + } else { + return nil, fmt.Errorf("invalid target gps %s", v) + } case "env": if vStr, ok := v.(string); ok { entries := strings.Split(vStr, ";") diff --git a/runner/benchmark/matrix_test.go b/runner/benchmark/matrix_test.go index ec3b05e4..41487656 100644 --- a/runner/benchmark/matrix_test.go +++ b/runner/benchmark/matrix_test.go @@ -88,6 +88,33 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { }, wantErr: false, }, + { + name: "config with target gps", + config: benchmark.TestDefinition{ + Variables: []benchmark.Param{ + { + ParamType: "payload", + Value: "load-test", + }, + { + ParamType: "target_gps", + Value: 200_000_000, + }, + }, + }, + want: []benchmark.TestRun{ + { + Params: types.RunParams{ + NodeType: "geth", + PayloadID: "load-test", + GasLimit: benchmark.DefaultParams.GasLimit, + TargetGPS: 200_000_000, + BlockTime: 1 * time.Second, + }, + }, + }, + wantErr: false, + }, { name: "duplicate param type", config: benchmark.TestDefinition{ @@ -182,6 +209,7 @@ func TestNewTestPlanFromConfigRoles(t *testing.T) { func TestNewTestPlanFromConfigDefaultsToBothRoles(t *testing.T) { config := &benchmark.BenchmarkConfig{Name: "test"} + definition := benchmark.TestDefinition{ Variables: []benchmark.Param{ { @@ -300,6 +328,45 @@ func TestNewTestPlanFromConfigAllowsSequencerThresholdsWithoutValidator(t *testi require.False(t, plan.Mode.RunValidator) } +func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { + config := &benchmark.BenchmarkConfig{Name: "snapshot load test"} + definition := benchmark.TestDefinition{ + Variables: []benchmark.Param{ + { + ParamType: "payload", + Value: "mainnet-snapshot-load-test", + }, + { + ParamType: "node_type", + Value: "builder", + }, + { + ParamType: "gas_limit", + Value: 1_200_000_000, + }, + { + ParamType: "target_gps", + Values: []interface{}{ + 80_000_000, + 400_000_000, + 1_200_000_000, + }, + }, + }, + } + + runs, err := benchmark.ResolveTestRunsFromMatrix(definition, "snapshot-load-test.yml", config) + require.NoError(t, err) + require.Len(t, runs, 3) + + require.Equal(t, uint64(1_200_000_000), runs[0].Params.GasLimit) + require.Equal(t, uint64(1_200_000_000), runs[1].Params.GasLimit) + require.Equal(t, uint64(1_200_000_000), runs[2].Params.GasLimit) + require.Equal(t, uint64(80_000_000), runs[0].Params.TargetGPS) + require.Equal(t, uint64(400_000_000), runs[1].Params.TargetGPS) + require.Equal(t, uint64(1_200_000_000), runs[2].Params.TargetGPS) +} + func stringPtr(s string) *string { return &s } diff --git a/runner/network/types/types.go b/runner/network/types/types.go index 9bb0b34a..02423c00 100644 --- a/runner/network/types/types.go +++ b/runner/network/types/types.go @@ -66,6 +66,9 @@ type RunParams struct { // GasLimit is the gas limit for the benchmark run which is the maximum gas that the sequencer will include per block. GasLimit uint64 + // TargetGPS is the target gas per second for load-test payloads. + TargetGPS uint64 + // PayloadID is a reference to a transaction payload that will be sent to the sequencer. PayloadID string @@ -112,6 +115,10 @@ func (p RunParams) ToConfig() map[string]interface{} { params["ValidatorNodeType"] = p.ValidatorNodeType } + if p.TargetGPS > 0 { + params["TargetGPS"] = p.TargetGPS + } + for k, v := range p.Tags { params[k] = v } diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index 92527aa8..114ea01e 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -25,7 +25,8 @@ import ( // LoadTestPayloadDefinition is the YAML payload params for the load-test type. // The load-test workload itself lives in a native base-load-tester config file; -// benchmark mode only overlays the RPC and timing fields it must control. +// benchmark mode overlays the RPC/timing fields it must control and overlays +// target_gps only when the benchmark matrix specifies one. type LoadTestPayloadDefinition struct { ConfigFile string `yaml:"config_file"` Network string `yaml:"network"` @@ -37,8 +38,7 @@ type loadTestPayloadWorker struct { loadTestBin string elRPCURL string flashblocksURL string - gasLimit uint64 - blockTimeSec uint64 + targetGPS uint64 params LoadTestPayloadDefinition mempool *mempool.StaticWorkloadMempool proxyServer *proxy.ProxyServer @@ -67,11 +67,6 @@ func NewLoadTestPayloadWorker( mp := mempool.NewStaticWorkloadMempool(log, chainID) ps := proxy.NewProxyServer(elRPCURL, log, cfg.ProxyPort(), mp) - blockTimeSec := uint64(params.BlockTime.Seconds()) - if blockTimeSec == 0 { - blockTimeSec = 1 - } - sourceConfigPath, err := resolveConfigFilePath(cfg.ConfigPath(), definition.ConfigFile) if err != nil { return nil, err @@ -83,8 +78,7 @@ func NewLoadTestPayloadWorker( loadTestBin: cfg.LoadTestBinary(), elRPCURL: elRPCURL, flashblocksURL: flashblocksURL, - gasLimit: params.GasLimit, - blockTimeSec: blockTimeSec, + targetGPS: params.TargetGPS, params: definition, mempool: mp, proxyServer: ps, @@ -224,14 +218,6 @@ func resolveConfigFilePath(benchmarkConfigPath string, loadTestConfigPath string return filepath.Join(filepath.Dir(benchmarkConfigPath), loadTestConfigPath), nil } -func (w *loadTestPayloadWorker) targetGPS() uint64 { - blockTimeSec := w.blockTimeSec - if blockTimeSec == 0 { - blockTimeSec = 1 - } - return w.gasLimit / blockTimeSec -} - func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { data, err := os.ReadFile(w.sourceConfigPath) if err != nil { @@ -257,7 +243,9 @@ func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { flashblocksURL = "ws://localhost:7111" } setMappingValue(config, "flashblocks_ws", stringNode(flashblocksURL)) - setMappingValue(config, "target_gps", uintNode(w.targetGPS())) + if w.targetGPS > 0 { + setMappingValue(config, "target_gps", uintNode(w.targetGPS)) + } setMappingValue(config, "duration", stringNode("99999s")) return config, nil @@ -332,9 +320,7 @@ func (w *loadTestPayloadWorker) writeConfig() (string, error) { w.log.Info("Generated load-test config", "source_config", w.sourceConfigPath, - "target_gps", w.targetGPS(), - "gas_limit", w.gasLimit, - "block_time_sec", w.blockTimeSec, + "target_gps", w.targetGPS, ) return tmpFile.Name(), nil diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go index b69e8784..85726c73 100644 --- a/runner/payload/loadtest/load_test_worker_test.go +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -66,8 +66,7 @@ transactions: worker := &loadTestPayloadWorker{ flashblocksURL: "ws://benchmark-flashblocks.example", - gasLimit: 150_000_000, - blockTimeSec: 2, + targetGPS: 75_000_000, proxyServer: proxy.NewProxyServer("http://sequencer.example", nil, 18546, nil), sourceConfigPath: configPath, } @@ -110,6 +109,38 @@ transactions: } } +func TestBuildConfigPreservesNativeTargetGPSWhenBenchmarkTargetUnset(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "load-test.yaml") + err := os.WriteFile(configPath, []byte(` +transaction_submission_rpcs: + - "http://standalone-submitter.invalid" +query_rpc: "http://standalone-query.invalid" +flashblocks_ws: "ws://standalone-flashblocks.invalid" +target_gps: 123 +duration: "60s" +transactions: + - weight: 100 + type: transfer +`), 0644) + require.NoError(t, err) + + worker := &loadTestPayloadWorker{ + flashblocksURL: "ws://benchmark-flashblocks.example", + proxyServer: proxy.NewProxyServer("http://sequencer.example", nil, 18546, nil), + sourceConfigPath: configPath, + } + + config, err := worker.buildConfig() + require.NoError(t, err) + + encoded, err := yaml.Marshal(config) + require.NoError(t, err) + output := string(encoded) + + require.Contains(t, output, "target_gps: 123") + require.Contains(t, output, "duration: 99999s") +} + func TestResolveConfigFilePath(t *testing.T) { resolved, err := resolveConfigFilePath("/tmp/configs/benchmark.yml", "load-tests/mainnet.yaml") require.NoError(t, err) From 632b2325f1481aacb421cb5a8ef42b3664dbb2a3 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 20 May 2026 20:25:36 -0500 Subject: [PATCH 06/15] Add configurable consensus timing --- configs/README.md | 2 +- runner/benchmark/benchmark.go | 9 +++ runner/benchmark/matrix_test.go | 22 +++++++ runner/network/consensus/client.go | 2 + .../network/consensus/sequencer_consensus.go | 66 ++++++++++++++----- .../consensus/sequencer_consensus_test.go | 36 ++++++++++ runner/network/sequencer_benchmark.go | 19 ++++-- runner/network/types/types.go | 16 +++++ 8 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 runner/network/consensus/sequencer_consensus_test.go diff --git a/configs/README.md b/configs/README.md index 0825df30..07fb9608 100644 --- a/configs/README.md +++ b/configs/README.md @@ -77,7 +77,7 @@ benchmarks: - name: "Benchmark Name" description: "What this benchmark tests" variables: - - type: payload|node_type|num_blocks|gas_limit|target_gps + - type: payload|node_type|num_blocks|gas_limit|target_gps|consensus_timing value: single-value values: [array, of, values] # for matrix testing ``` diff --git a/runner/benchmark/benchmark.go b/runner/benchmark/benchmark.go index cd82985a..2e7631fd 100644 --- a/runner/benchmark/benchmark.go +++ b/runner/benchmark/benchmark.go @@ -76,6 +76,15 @@ func NewParamsFromValues(assignments map[string]interface{}) (*types.RunParams, } else { return nil, fmt.Errorf("invalid target gps %s", v) } + case "consensus_timing": + if vStr, ok := v.(string); ok { + if vStr != "" && vStr != types.ConsensusTimingModePreventLateFCU && vStr != types.ConsensusTimingModeBaseConsensus { + return nil, fmt.Errorf("invalid consensus timing %s", v) + } + params.ConsensusTimingMode = vStr + } else { + return nil, fmt.Errorf("invalid consensus timing %s", v) + } case "env": if vStr, ok := v.(string); ok { entries := strings.Split(vStr, ";") diff --git a/runner/benchmark/matrix_test.go b/runner/benchmark/matrix_test.go index 41487656..0d471bb6 100644 --- a/runner/benchmark/matrix_test.go +++ b/runner/benchmark/matrix_test.go @@ -352,6 +352,10 @@ func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { 1_200_000_000, }, }, + { + ParamType: "consensus_timing", + Value: types.ConsensusTimingModeBaseConsensus, + }, }, } @@ -365,6 +369,24 @@ func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { require.Equal(t, uint64(80_000_000), runs[0].Params.TargetGPS) require.Equal(t, uint64(400_000_000), runs[1].Params.TargetGPS) require.Equal(t, uint64(1_200_000_000), runs[2].Params.TargetGPS) + require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[0].Params.ConsensusTimingMode) + require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[1].Params.ConsensusTimingMode) + require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[2].Params.ConsensusTimingMode) +} + +func TestResolveTestRunsFromMatrixRejectsInvalidConsensusTiming(t *testing.T) { + config := &benchmark.BenchmarkConfig{Name: "benchmark"} + definition := benchmark.TestDefinition{ + Variables: []benchmark.Param{ + { + ParamType: "consensus_timing", + Value: "aligned", + }, + }, + } + + _, err := benchmark.ResolveTestRunsFromMatrix(definition, "benchmark.yml", config) + require.ErrorContains(t, err, "invalid consensus timing") } func stringPtr(s string) *string { diff --git a/runner/network/consensus/client.go b/runner/network/consensus/client.go index a2308230..76cb5d8b 100644 --- a/runner/network/consensus/client.go +++ b/runner/network/consensus/client.go @@ -23,6 +23,8 @@ type ConsensusClientOptions struct { GasLimitSetup uint64 // ParallelTxBatches is the number of parallel batches for sending transactions ParallelTxBatches int + // ConsensusTimingMode controls how FCU and getPayload calls are scheduled. + ConsensusTimingMode string } // BaseConsensusClient contains common functionality shared between different consensus client implementations. diff --git a/runner/network/consensus/sequencer_consensus.go b/runner/network/consensus/sequencer_consensus.go index 2e2ed3c9..81faaed4 100644 --- a/runner/network/consensus/sequencer_consensus.go +++ b/runner/network/consensus/sequencer_consensus.go @@ -112,7 +112,7 @@ func marshalBinaryWithSignature(info *derive.L1BlockInfo, signature []byte) ([]b return w.Bytes(), nil } -func (f *SequencerConsensusClient) generatePayloadAttributes(sequencerTxs [][]byte, isSetupPayload bool, nextBoundary time.Time, blockTime time.Duration) (*eth.PayloadAttributes, *common.Hash, error) { +func (f *SequencerConsensusClient) generatePayloadAttributes(sequencerTxs [][]byte, isSetupPayload bool, timestamp uint64) (*eth.PayloadAttributes, *common.Hash, error) { gasLimit := eth.Uint64Quantity(f.options.GasLimit) if isSetupPayload { gasLimit = eth.Uint64Quantity(f.options.GasLimitSetup) @@ -121,12 +121,6 @@ func (f *SequencerConsensusClient) generatePayloadAttributes(sequencerTxs [][]by var b8 eth.Bytes8 copy(b8[:], eip1559.EncodeHolocene1559Params(50, 1)) - // Use nextBoundary (the block-time-aligned wall-clock boundary we slept to) plus - // one block time as the block timestamp. This guarantees the FCU always arrives - // at the same point relative to the block deadline, eliminating the jitter that - // causes "FCU arrived too late" and empty blocks when sendTxs takes variable time. - timestamp := uint64(nextBoundary.Add(blockTime).Unix()) - number := uint64(0) time := uint64(0) baseFee := big.NewInt(1) @@ -210,6 +204,29 @@ func (f *SequencerConsensusClient) generatePayloadAttributes(sequencerTxs [][]by return payloadAttrs, &root, nil } +func nextPayloadTimestamp(lastTimestamp uint64, now time.Time, blockTime time.Duration) uint64 { + blockTimeSeconds := uint64(blockTime / time.Second) + if blockTimeSeconds == 0 { + blockTimeSeconds = 1 + } + + // Match the sequencer cadence when possible: the next payload timestamp is + // one block time after the parent. If transaction draining made that slot + // too close to wall clock, skip ahead so the builder still has time to work. + timestamp := lastTimestamp + blockTimeSeconds + minLead := blockTime / 2 + if minLead <= 0 { + minLead = time.Second + } + + minDeadline := now.Add(minLead) + for time.Unix(int64(timestamp), 0).Before(minDeadline) { + timestamp += blockTimeSeconds + } + + return timestamp +} + // Propose starts block generation, waits BlockTime, and generates a block. func (f *SequencerConsensusClient) Propose(ctx context.Context, blockMetrics *metrics.BlockMetrics, isSetupPayload bool) (*engine.ExecutableData, error) { startTime := time.Now() @@ -265,16 +282,32 @@ func (f *SequencerConsensusClient) Propose(ctx context.Context, blockMetrics *me f.log.Info("Sent transactions", "duration", duration, "num_txs", len(sendTxs)) blockMetrics.AddExecutionMetric(networktypes.SendTxsLatencyMetric, duration) - now := time.Now() - nextBoundary := now.Truncate(f.options.BlockTime).Add(f.options.BlockTime) - sleepDuration := time.Until(nextBoundary) - f.log.Info("Aligning to next block time boundary before FCU", "sleep", sleepDuration, "block_time", f.options.BlockTime) - time.Sleep(sleepDuration) - startBlockBuildingTime := time.Now() + var payloadTimestamp uint64 + var blockDeadline time.Time + if f.options.ConsensusTimingMode == networktypes.ConsensusTimingModeBaseConsensus { + payloadTimestamp = nextPayloadTimestamp(f.lastTimestamp, startBlockBuildingTime, f.options.BlockTime) + blockDeadline = time.Unix(int64(payloadTimestamp), 0) + } else { + nextBoundary := startBlockBuildingTime.Truncate(f.options.BlockTime).Add(f.options.BlockTime) + sleepDuration := time.Until(nextBoundary) + f.log.Info("Aligning to next block time boundary before FCU", "sleep", sleepDuration, "block_time", f.options.BlockTime) + if sleepDuration > 0 { + time.Sleep(sleepDuration) + } + startBlockBuildingTime = time.Now() + + // Use nextBoundary (the block-time-aligned wall-clock boundary we slept to) plus + // one block time as the block timestamp. This guarantees the FCU always arrives + // at the same point relative to the block deadline, eliminating the jitter that + // causes "FCU arrived too late" and empty blocks when sendTxs takes variable time. + blockDeadline = nextBoundary.Add(f.options.BlockTime) + payloadTimestamp = uint64(blockDeadline.Unix()) + } + f.log.Info("Starting block building") - payloadAttrs, beaconRoot, err := f.generatePayloadAttributes(sequencerTxs, isSetupPayload, nextBoundary, f.options.BlockTime) + payloadAttrs, beaconRoot, err := f.generatePayloadAttributes(sequencerTxs, isSetupPayload, payloadTimestamp) if err != nil { return nil, errors.Wrap(err, "failed to generate payload attributes") } @@ -292,10 +325,11 @@ func (f *SequencerConsensusClient) Propose(ctx context.Context, blockMetrics *me blockMetrics.AddExecutionMetric(networktypes.UpdateForkChoiceLatencyMetric, duration) f.currentPayloadID = payloadID - blockDeadline := nextBoundary.Add(f.options.BlockTime) waitDuration := time.Until(blockDeadline) f.log.Info("Waiting for block deadline", "wait", waitDuration) - time.Sleep(waitDuration) + if waitDuration > 0 { + time.Sleep(waitDuration) + } startTime = time.Now() diff --git a/runner/network/consensus/sequencer_consensus_test.go b/runner/network/consensus/sequencer_consensus_test.go new file mode 100644 index 00000000..54c8421f --- /dev/null +++ b/runner/network/consensus/sequencer_consensus_test.go @@ -0,0 +1,36 @@ +package consensus + +import ( + "testing" + "time" +) + +func TestNextPayloadTimestampUsesNextBlockTime(t *testing.T) { + now := time.Unix(100, int64(100*time.Millisecond)) + + timestamp := nextPayloadTimestamp(100, now, 2*time.Second) + + if timestamp != 102 { + t.Fatalf("expected next payload timestamp 102, got %d", timestamp) + } +} + +func TestNextPayloadTimestampSkipsTooCloseSlot(t *testing.T) { + now := time.Unix(101, int64(250*time.Millisecond)) + + timestamp := nextPayloadTimestamp(100, now, 2*time.Second) + + if timestamp != 104 { + t.Fatalf("expected next payload timestamp 104, got %d", timestamp) + } +} + +func TestNextPayloadTimestampCatchesUpFromWallClock(t *testing.T) { + now := time.Unix(120, 0) + + timestamp := nextPayloadTimestamp(100, now, 2*time.Second) + + if timestamp != 122 { + t.Fatalf("expected next payload timestamp 122, got %d", timestamp) + } +} diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index 83e493c4..ac3c436b 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -207,10 +207,11 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. go func() { consensusClient := consensus.NewSequencerConsensusClient(nb.log, sequencerClient.Client(), sequencerClient.AuthClient(), mempool, consensus.ConsensusClientOptions{ - BlockTime: params.BlockTime, - GasLimit: params.GasLimit, - GasLimitSetup: 1e9, // 1G gas - ParallelTxBatches: nb.config.Config.ParallelTxBatches(), + BlockTime: params.BlockTime, + GasLimit: params.GasLimit, + GasLimitSetup: 1e9, // 1G gas + ParallelTxBatches: nb.config.Config.ParallelTxBatches(), + ConsensusTimingMode: params.ConsensusTimingMode, }, headBlockHash, headBlockNumber, l1Chain, nb.config.BatcherAddr()) payloads := make([]engine.ExecutableData, 0) @@ -275,8 +276,10 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. pendingTxs = 0 } - log.Info("Sleeping for block time", "block_time", params.BlockTime) - time.Sleep(params.BlockTime) + if !params.UseBaseConsensusTiming() { + log.Info("Sleeping for block time", "block_time", params.BlockTime) + time.Sleep(params.BlockTime) + } err = metricsCollector.Collect(benchmarkCtx, blockMetrics) if err != nil { @@ -370,7 +373,9 @@ func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( } settlementBlock++ - time.Sleep(nb.config.Params.BlockTime) + if !nb.config.Params.UseBaseConsensusTiming() { + time.Sleep(nb.config.Params.BlockTime) + } } } diff --git a/runner/network/types/types.go b/runner/network/types/types.go index 02423c00..8f78d9e7 100644 --- a/runner/network/types/types.go +++ b/runner/network/types/types.go @@ -84,6 +84,9 @@ type RunParams struct { // BlockTime is the time between blocks in the benchmark run. BlockTime time.Duration + // ConsensusTimingMode controls how the fake consensus client schedules FCU/getPayload calls. + ConsensusTimingMode string + // Env is the environment variables for the benchmark run. Env map[string]string @@ -100,6 +103,15 @@ type RunParams struct { ClientBinPath string } +const ( + ConsensusTimingModePreventLateFCU = "prevent-late-fcu" + ConsensusTimingModeBaseConsensus = "base-consensus" +) + +func (p RunParams) UseBaseConsensusTiming() bool { + return p.ConsensusTimingMode == ConsensusTimingModeBaseConsensus +} + func (p RunParams) ToConfig() map[string]interface{} { params := map[string]interface{}{ "NodeType": p.NodeType, @@ -119,6 +131,10 @@ func (p RunParams) ToConfig() map[string]interface{} { params["TargetGPS"] = p.TargetGPS } + if p.ConsensusTimingMode != "" { + params["ConsensusTimingMode"] = p.ConsensusTimingMode + } + for k, v := range p.Tags { params[k] = v } From ea3230b49b2725f798025a6a3d56e64f3407eb1b Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 20 May 2026 21:25:43 -0500 Subject: [PATCH 07/15] Default snapshot load tests to base-consensus timing --- configs/README.md | 2 + runner/benchmark/matrix.go | 25 +++++++ runner/benchmark/matrix_test.go | 125 ++++++++++++++++++++++++-------- 3 files changed, 121 insertions(+), 31 deletions(-) diff --git a/configs/README.md b/configs/README.md index 07fb9608..64970fe8 100644 --- a/configs/README.md +++ b/configs/README.md @@ -82,6 +82,8 @@ benchmarks: values: [array, of, values] # for matrix testing ``` +`consensus_timing` can be `prevent-late-fcu` or `base-consensus`. Snapshot load-test runs default to `base-consensus`; other benchmark runs default to `prevent-late-fcu`. + ## 🎯 Choosing the Right Configuration - **Development/Testing**: Use `examples/` configurations for focused testing diff --git a/runner/benchmark/matrix.go b/runner/benchmark/matrix.go index 180f78f1..46832457 100644 --- a/runner/benchmark/matrix.go +++ b/runner/benchmark/matrix.go @@ -3,6 +3,8 @@ package benchmark import ( "fmt" "time" + + "github.com/base/base-bench/runner/network/types" ) type ThresholdConfig struct { @@ -134,6 +136,7 @@ func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *Be if c.Tags != nil { params.Tags = *c.Tags } + params.ConsensusTimingMode = consensusTimingMode(params, c, config) testParams[i] = TestRun{ ID: id, @@ -166,3 +169,25 @@ func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *Be return testParams, nil } + +func consensusTimingMode(params *types.RunParams, definition TestDefinition, config *BenchmarkConfig) string { + if params.ConsensusTimingMode != "" { + return params.ConsensusTimingMode + } + if isSnapshotLoadTest(params.PayloadID, definition, config) { + return types.ConsensusTimingModeBaseConsensus + } + return types.ConsensusTimingModePreventLateFCU +} + +func isSnapshotLoadTest(payloadID string, definition TestDefinition, config *BenchmarkConfig) bool { + if definition.Snapshot == nil || definition.Snapshot.Command == "" { + return false + } + for _, transactionPayload := range config.TransactionPayloads { + if transactionPayload.ID == payloadID && transactionPayload.Type == "load-test" { + return true + } + } + return false +} diff --git a/runner/benchmark/matrix_test.go b/runner/benchmark/matrix_test.go index 0d471bb6..adb99c4e 100644 --- a/runner/benchmark/matrix_test.go +++ b/runner/benchmark/matrix_test.go @@ -6,6 +6,7 @@ import ( "github.com/base/base-bench/runner/benchmark" "github.com/base/base-bench/runner/network/types" + "github.com/base/base-bench/runner/payload" "github.com/stretchr/testify/require" ) @@ -29,10 +30,11 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { want: []benchmark.TestRun{ { Params: types.RunParams{ - NodeType: "geth", - PayloadID: "simple", - GasLimit: benchmark.DefaultParams.GasLimit, - BlockTime: 1 * time.Second, + NodeType: "geth", + PayloadID: "simple", + GasLimit: benchmark.DefaultParams.GasLimit, + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, }, @@ -55,34 +57,38 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { want: []benchmark.TestRun{ { Params: types.RunParams{ - NodeType: "geth", - GasLimit: benchmark.DefaultParams.GasLimit, - PayloadID: "simple", - BlockTime: 1 * time.Second, + NodeType: "geth", + GasLimit: benchmark.DefaultParams.GasLimit, + PayloadID: "simple", + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, { Params: types.RunParams{ - NodeType: "erigon", - GasLimit: benchmark.DefaultParams.GasLimit, - PayloadID: "simple", - BlockTime: 1 * time.Second, + NodeType: "erigon", + GasLimit: benchmark.DefaultParams.GasLimit, + PayloadID: "simple", + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, { Params: types.RunParams{ - NodeType: "geth", - GasLimit: benchmark.DefaultParams.GasLimit, - PayloadID: "complex", - BlockTime: 1 * time.Second, + NodeType: "geth", + GasLimit: benchmark.DefaultParams.GasLimit, + PayloadID: "complex", + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, { Params: types.RunParams{ - NodeType: "erigon", - GasLimit: benchmark.DefaultParams.GasLimit, - PayloadID: "complex", - BlockTime: 1 * time.Second, + NodeType: "erigon", + GasLimit: benchmark.DefaultParams.GasLimit, + PayloadID: "complex", + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, }, @@ -105,11 +111,12 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { want: []benchmark.TestRun{ { Params: types.RunParams{ - NodeType: "geth", - PayloadID: "load-test", - GasLimit: benchmark.DefaultParams.GasLimit, - TargetGPS: 200_000_000, - BlockTime: 1 * time.Second, + NodeType: "geth", + PayloadID: "load-test", + GasLimit: benchmark.DefaultParams.GasLimit, + TargetGPS: 200_000_000, + BlockTime: 1 * time.Second, + ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, }, }, }, @@ -352,10 +359,6 @@ func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { 1_200_000_000, }, }, - { - ParamType: "consensus_timing", - Value: types.ConsensusTimingModeBaseConsensus, - }, }, } @@ -369,9 +372,69 @@ func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { require.Equal(t, uint64(80_000_000), runs[0].Params.TargetGPS) require.Equal(t, uint64(400_000_000), runs[1].Params.TargetGPS) require.Equal(t, uint64(1_200_000_000), runs[2].Params.TargetGPS) + require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[0].Params.ConsensusTimingMode) + require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[1].Params.ConsensusTimingMode) + require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[2].Params.ConsensusTimingMode) +} + +func TestResolveTestRunsFromMatrixDefaultsSnapshotLoadTestsToBaseConsensusTiming(t *testing.T) { + config := &benchmark.BenchmarkConfig{ + Name: "snapshot load test", + TransactionPayloads: []payload.Definition{ + { + ID: "mainnet-snapshot-load-test", + Type: "load-test", + }, + }, + } + definition := benchmark.TestDefinition{ + Snapshot: &benchmark.SnapshotDefinition{ + Command: "./setup-initial-snapshot.sh --network mainnet", + }, + Variables: []benchmark.Param{ + { + ParamType: "payload", + Value: "mainnet-snapshot-load-test", + }, + }, + } + + runs, err := benchmark.ResolveTestRunsFromMatrix(definition, "snapshot-load-test.yml", config) + require.NoError(t, err) + require.Len(t, runs, 1) require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[0].Params.ConsensusTimingMode) - require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[1].Params.ConsensusTimingMode) - require.Equal(t, types.ConsensusTimingModeBaseConsensus, runs[2].Params.ConsensusTimingMode) +} + +func TestResolveTestRunsFromMatrixAllowsSnapshotLoadTestsToOverrideConsensusTiming(t *testing.T) { + config := &benchmark.BenchmarkConfig{ + Name: "snapshot load test", + TransactionPayloads: []payload.Definition{ + { + ID: "mainnet-snapshot-load-test", + Type: "load-test", + }, + }, + } + definition := benchmark.TestDefinition{ + Snapshot: &benchmark.SnapshotDefinition{ + Command: "./setup-initial-snapshot.sh --network mainnet", + }, + Variables: []benchmark.Param{ + { + ParamType: "payload", + Value: "mainnet-snapshot-load-test", + }, + { + ParamType: "consensus_timing", + Value: types.ConsensusTimingModePreventLateFCU, + }, + }, + } + + runs, err := benchmark.ResolveTestRunsFromMatrix(definition, "snapshot-load-test.yml", config) + require.NoError(t, err) + require.Len(t, runs, 1) + require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[0].Params.ConsensusTimingMode) } func TestResolveTestRunsFromMatrixRejectsInvalidConsensusTiming(t *testing.T) { From 7455db139087a89f312267d560e6afac2109c566 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 20 May 2026 23:40:03 -0500 Subject: [PATCH 08/15] Submit load tests through the node txpool --- runner/payload/loadtest/load_test_worker.go | 27 +++++-------------- .../payload/loadtest/load_test_worker_test.go | 9 +++---- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index 114ea01e..7e26c062 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/base/base-bench/runner/clients/common/proxy" "github.com/base/base-bench/runner/config" "github.com/base/base-bench/runner/network/mempool" "github.com/base/base-bench/runner/network/types" @@ -41,7 +40,6 @@ type loadTestPayloadWorker struct { targetGPS uint64 params LoadTestPayloadDefinition mempool *mempool.StaticWorkloadMempool - proxyServer *proxy.ProxyServer cmd *exec.Cmd done chan struct{} shutdownOnce sync.Once @@ -51,7 +49,7 @@ type loadTestPayloadWorker struct { } // NewLoadTestPayloadWorker creates a worker that runs the base-load-test binary -// as an external transaction generator, capturing transactions via a proxy server. +// as an external transaction generator against the benchmark node's RPC. func NewLoadTestPayloadWorker( log log.Logger, elRPCURL string, @@ -65,7 +63,6 @@ func NewLoadTestPayloadWorker( outputPath string, ) (worker.Worker, error) { mp := mempool.NewStaticWorkloadMempool(log, chainID) - ps := proxy.NewProxyServer(elRPCURL, log, cfg.ProxyPort(), mp) sourceConfigPath, err := resolveConfigFilePath(cfg.ConfigPath(), definition.ConfigFile) if err != nil { @@ -81,7 +78,6 @@ func NewLoadTestPayloadWorker( targetGPS: params.TargetGPS, params: definition, mempool: mp, - proxyServer: ps, sourceConfigPath: sourceConfigPath, outputPath: outputPath, } @@ -94,10 +90,6 @@ func (w *loadTestPayloadWorker) Mempool() mempool.FakeMempool { } func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { - if err := w.proxyServer.Run(ctx); err != nil { - return errors.Wrap(err, "failed to run proxy server") - } - configPath, err := w.writeConfig() if err != nil { return errors.Wrap(err, "failed to write load-test config") @@ -189,8 +181,6 @@ func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { } } - w.proxyServer.Stop() - if w.renderedConfigPath != "" { if err := os.Remove(w.renderedConfigPath); err != nil { w.log.Warn("failed to remove load-test config", "path", w.renderedConfigPath, "err", err) @@ -200,12 +190,8 @@ func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { return nil } -func (w *loadTestPayloadWorker) SendTxs(ctx context.Context, _ int) (int, error) { - w.log.Info("Collecting txs from load test") - pendingTxs := w.proxyServer.DrainPendingTxs() - - w.mempool.AddTransactions(pendingTxs) - return len(pendingTxs), nil +func (w *loadTestPayloadWorker) SendTxs(_ context.Context, _ int) (int, error) { + return 0, nil } func resolveConfigFilePath(benchmarkConfigPath string, loadTestConfigPath string) (string, error) { @@ -234,9 +220,8 @@ func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { return nil, err } - proxyURL := w.proxyServer.ClientURL() - setMappingValue(config, "transaction_submission_rpcs", stringSequenceNode(proxyURL)) - setMappingValue(config, "query_rpc", stringNode(proxyURL)) + setMappingValue(config, "transaction_submission_rpcs", stringSequenceNode(w.elRPCURL)) + setMappingValue(config, "query_rpc", stringNode(w.elRPCURL)) flashblocksURL := w.flashblocksURL if flashblocksURL == "" { @@ -293,7 +278,7 @@ func stringSequenceNode(values ...string) *yaml.Node { } // writeConfig generates a temporary YAML config file for the load-test binary -// with the RPC URL pointing to the proxy server. +// with benchmark-controlled RPC, timing, and report fields. func (w *loadTestPayloadWorker) writeConfig() (string, error) { config, err := w.buildConfig() if err != nil { diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go index 85726c73..388e2bf9 100644 --- a/runner/payload/loadtest/load_test_worker_test.go +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -5,7 +5,6 @@ import ( "path/filepath" "testing" - "github.com/base/base-bench/runner/clients/common/proxy" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -67,7 +66,7 @@ transactions: worker := &loadTestPayloadWorker{ flashblocksURL: "ws://benchmark-flashblocks.example", targetGPS: 75_000_000, - proxyServer: proxy.NewProxyServer("http://sequencer.example", nil, 18546, nil), + elRPCURL: "http://sequencer.example", sourceConfigPath: configPath, } @@ -79,8 +78,8 @@ transactions: output := string(encoded) for _, want := range []string{ - "transaction_submission_rpcs:\n - http://localhost:18546", - "query_rpc: http://localhost:18546", + "transaction_submission_rpcs:\n - http://sequencer.example", + "query_rpc: http://sequencer.example", "flashblocks_ws: ws://benchmark-flashblocks.example", "target_gps: 75000000", "duration: 99999s", @@ -126,7 +125,7 @@ transactions: worker := &loadTestPayloadWorker{ flashblocksURL: "ws://benchmark-flashblocks.example", - proxyServer: proxy.NewProxyServer("http://sequencer.example", nil, 18546, nil), + elRPCURL: "http://sequencer.example", sourceConfigPath: configPath, } From 03ac2c744a54363041f5cf6cadcdfac538998ba2 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 20 May 2026 23:51:44 -0500 Subject: [PATCH 09/15] Run load tests until completion --- runner/network/sequencer_benchmark.go | 142 ++++++++++++------ runner/payload/loadtest/load_test_worker.go | 18 ++- .../payload/loadtest/load_test_worker_test.go | 5 +- runner/payload/worker/types.go | 7 + 4 files changed, 121 insertions(+), 51 deletions(-) diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index ac3c436b..45c0fd82 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -240,57 +240,63 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. payloads = append(payloads, *lastSetupPayload) - blockMetrics := metrics.NewBlockMetrics() - pendingTxs := 0 + completionWorker, runUntilWorkerDone := transactionWorker.(payloadworker.CompletionWorker) + if runUntilWorkerDone { + nb.log.Info("Running benchmark blocks until payload worker completes") + blockIndex := uint64(1) + runUntilDoneLoop: + for { + select { + case <-completionWorker.Done(): + if err := completionWorker.Err(); err != nil { + errChan <- errors.Wrap(err, "payload worker failed") + return + } + nb.log.Info("Payload worker completed", "blocks", blockIndex-1) + break runUntilDoneLoop + default: + } - // run for a few blocks - for i := 0; i < params.NumBlocks; i++ { - blockMetrics.SetBlockNumber(uint64(i) + 1) - txsSent, err := transactionWorker.SendTxs(benchmarkCtx, pendingTxs) - if err != nil { - nb.log.Warn("failed to send transactions", "err", err) - errChan <- err - return + payload, updatedPendingTxs, err := nb.proposeBenchmarkBlock( + benchmarkCtx, + transactionWorker, + consensusClient, + metricsCollector, + blockIndex, + pendingTxs, + ) + if err != nil { + errChan <- err + return + } + pendingTxs = updatedPendingTxs + payloads = append(payloads, *payload) + blockIndex++ } - - payload, err := consensusClient.Propose(benchmarkCtx, blockMetrics, false) - if err != nil { - errChan <- err - return + } else { + // run for a few blocks + for i := 0; i < params.NumBlocks; i++ { + payload, updatedPendingTxs, err := nb.proposeBenchmarkBlock( + benchmarkCtx, + transactionWorker, + consensusClient, + metricsCollector, + uint64(i)+1, + pendingTxs, + ) + if err != nil { + errChan <- err + return + } + pendingTxs = updatedPendingTxs + payloads = append(payloads, *payload) } - if payload == nil { - errChan <- errors.New("received nil payload from consensus client") + if err := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs); err != nil { + errChan <- err return } - - // Track how many user txs are still pending in the node's mempool. - // payload.Transactions includes the L1 info deposit tx, so user txs = total - 1. - userTxsIncluded := len(payload.Transactions) - 1 - if userTxsIncluded < 0 { - userTxsIncluded = 0 - } - pendingTxs = pendingTxs + txsSent - userTxsIncluded - if pendingTxs < 0 { - pendingTxs = 0 - } - - if !params.UseBaseConsensusTiming() { - log.Info("Sleeping for block time", "block_time", params.BlockTime) - time.Sleep(params.BlockTime) - } - - err = metricsCollector.Collect(benchmarkCtx, blockMetrics) - if err != nil { - nb.log.Error("Failed to collect metrics", "error", err) - } - payloads = append(payloads, *payload) - } - - if err := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs); err != nil { - errChan <- err - return } if err := consensusClient.Stop(benchmarkCtx); err != nil { @@ -319,6 +325,54 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. } } +func (nb *sequencerBenchmark) proposeBenchmarkBlock( + ctx context.Context, + transactionWorker payloadworker.Worker, + consensusClient *consensus.SequencerConsensusClient, + metricsCollector metrics.Collector, + blockIndex uint64, + pendingTxs int, +) (*engine.ExecutableData, int, error) { + blockMetrics := metrics.NewBlockMetrics() + blockMetrics.SetBlockNumber(blockIndex) + + txsSent, err := transactionWorker.SendTxs(ctx, pendingTxs) + if err != nil { + nb.log.Warn("failed to send transactions", "err", err) + return nil, pendingTxs, err + } + + payload, err := consensusClient.Propose(ctx, blockMetrics, false) + if err != nil { + return nil, pendingTxs, err + } + if payload == nil { + return nil, pendingTxs, errors.New("received nil payload from consensus client") + } + + // Track how many user txs are still pending in the node's mempool. + // payload.Transactions includes the L1 info deposit tx, so user txs = total - 1. + userTxsIncluded := len(payload.Transactions) - 1 + if userTxsIncluded < 0 { + userTxsIncluded = 0 + } + updatedPendingTxs := pendingTxs + txsSent - userTxsIncluded + if updatedPendingTxs < 0 { + updatedPendingTxs = 0 + } + + if !nb.config.Params.UseBaseConsensusTiming() { + log.Info("Sleeping for block time", "block_time", nb.config.Params.BlockTime) + time.Sleep(nb.config.Params.BlockTime) + } + + if err := metricsCollector.Collect(ctx, blockMetrics); err != nil { + nb.log.Error("Failed to collect metrics", "error", err) + } + + return payload, updatedPendingTxs, nil +} + func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( ctx context.Context, transactionWorker payloadworker.Worker, diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index 7e26c062..ecb557cb 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -24,8 +24,8 @@ import ( // LoadTestPayloadDefinition is the YAML payload params for the load-test type. // The load-test workload itself lives in a native base-load-tester config file; -// benchmark mode overlays the RPC/timing fields it must control and overlays -// target_gps only when the benchmark matrix specifies one. +// benchmark mode overlays the RPC fields it must control and overlays target_gps +// only when the benchmark matrix specifies one. type LoadTestPayloadDefinition struct { ConfigFile string `yaml:"config_file"` Network string `yaml:"network"` @@ -43,6 +43,8 @@ type loadTestPayloadWorker struct { cmd *exec.Cmd done chan struct{} shutdownOnce sync.Once + waitErrMu sync.Mutex + waitErr error sourceConfigPath string renderedConfigPath string outputPath string @@ -115,7 +117,10 @@ func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { w.cmd = cmd w.done = make(chan struct{}) go func() { - _ = cmd.Wait() + err := cmd.Wait() + w.waitErrMu.Lock() + w.waitErr = err + w.waitErrMu.Unlock() close(w.done) }() @@ -160,6 +165,12 @@ func (w *loadTestPayloadWorker) Done() <-chan struct{} { return done } +func (w *loadTestPayloadWorker) Err() error { + w.waitErrMu.Lock() + defer w.waitErrMu.Unlock() + return w.waitErr +} + func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { if w.cmd != nil && w.cmd.Process != nil { if err := w.BeginGracefulShutdown(ctx); err != nil { @@ -231,7 +242,6 @@ func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { if w.targetGPS > 0 { setMappingValue(config, "target_gps", uintNode(w.targetGPS)) } - setMappingValue(config, "duration", stringNode("99999s")) return config, nil } diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go index 388e2bf9..7bef24bb 100644 --- a/runner/payload/loadtest/load_test_worker_test.go +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -82,7 +82,7 @@ transactions: "query_rpc: http://sequencer.example", "flashblocks_ws: ws://benchmark-flashblocks.example", "target_gps: 75000000", - "duration: 99999s", + "duration: \"60s\"", "chain_id: 8453", "sender_count: 250", "in_flight_per_sender: 64", @@ -102,7 +102,6 @@ transactions: "standalone-query.invalid", "standalone-flashblocks.invalid", "target_gps: 123", - "duration: 60s", } { require.NotContains(t, output, oldValue) } @@ -137,7 +136,7 @@ transactions: output := string(encoded) require.Contains(t, output, "target_gps: 123") - require.Contains(t, output, "duration: 99999s") + require.Contains(t, output, "duration: \"60s\"") } func TestResolveConfigFilePath(t *testing.T) { diff --git a/runner/payload/worker/types.go b/runner/payload/worker/types.go index a0e05725..942df001 100644 --- a/runner/payload/worker/types.go +++ b/runner/payload/worker/types.go @@ -25,3 +25,10 @@ type GracefulShutdownWorker interface { BeginGracefulShutdown(ctx context.Context) error Done() <-chan struct{} } + +// CompletionWorker owns its own run duration. The benchmark sequencer keeps +// producing blocks until Done closes, then treats Err as the worker result. +type CompletionWorker interface { + Done() <-chan struct{} + Err() error +} From 953f258e1c713f99dadaba8fee3731a3836e2621 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Thu, 21 May 2026 15:40:27 -0500 Subject: [PATCH 10/15] Expose snapshot load test benchmark metrics --- report/src/components/ChartGrid.tsx | 16 ++ report/src/components/ConfigurationTags.tsx | 87 ++++++--- report/src/components/RunList.tsx | 6 +- report/src/metricDefinitions.ts | 177 ++++++++++++++++++ report/src/pages/RunIndex.tsx | 7 +- runner/clients/builder/metrics.go | 189 ++++++++++++++++++-- runner/clients/builder/metrics_test.go | 98 ++++++++++ runner/metrics/metrics_interface.go | 15 ++ 8 files changed, 557 insertions(+), 38 deletions(-) create mode 100644 runner/clients/builder/metrics_test.go diff --git a/report/src/components/ChartGrid.tsx b/report/src/components/ChartGrid.tsx index bd525329..d339d146 100644 --- a/report/src/components/ChartGrid.tsx +++ b/report/src/components/ChartGrid.tsx @@ -20,6 +20,22 @@ function resolveMetricKey( return key; } } + + const metricKeys = chartData.flatMap((d) => Object.keys(d.ExecutionMetrics)); + for (const key of keys) { + const quantileSuffix = key.match(/(_quantile_\d+(?:_\d+)?)$/)?.[1] ?? ""; + const metricPrefix = quantileSuffix + ? key.slice(0, -quantileSuffix.length) + : key; + const labeledMetricKey = metricKeys.find( + (metricKey) => + metricKey.startsWith(`${metricPrefix}_`) && + (!quantileSuffix || metricKey.endsWith(quantileSuffix)), + ); + if (labeledMetricKey) { + return labeledMetricKey; + } + } return primaryKey; } diff --git a/report/src/components/ConfigurationTags.tsx b/report/src/components/ConfigurationTags.tsx index 5d3591b7..5568120b 100644 --- a/report/src/components/ConfigurationTags.tsx +++ b/report/src/components/ConfigurationTags.tsx @@ -7,6 +7,61 @@ interface ConfigurationTagsProps { className?: string; } +const CONFIG_LABELS: Record = { + BlockTimeMilliseconds: "Block Time", + ConsensusTimingMode: "Consensus Timing", + GasLimit: "Gas Limit", + NodeType: "Node Type", + TargetGPS: "Target Gas/s", + TransactionPayload: "Transaction Payload", + ValidatorNodeType: "Validator Node Type", +}; + +const CONFIG_ORDER = [ + "TargetGPS", + "GasLimit", + "BlockTimeMilliseconds", + "ConsensusTimingMode", + "NodeType", + "ValidatorNodeType", + "TransactionPayload", + "Roles", +]; + +const configLabel = (key: string): string => + CONFIG_LABELS[key] ?? camelToTitleCase(key); + +const configValue = (key: string, value: unknown): string => { + if (key === "GasLimit") { + return formatValue(Number(value), "gas"); + } + if (key === "TargetGPS") { + return formatValue(Number(value), "gas/s"); + } + if (key === "BlockTimeMilliseconds") { + return formatValue(Number(value), "ms"); + } + return String(formatLabel(`${value}`)); +}; + +const configEntries = (testConfig: Record) => + Object.entries(testConfig || {}) + .filter(([key, value]) => key !== "BenchmarkRun" && value !== "") + .sort(([a], [b]) => { + const aIndex = CONFIG_ORDER.indexOf(a); + const bIndex = CONFIG_ORDER.indexOf(b); + if (aIndex === -1 && bIndex === -1) { + return a.localeCompare(b); + } + if (aIndex === -1) { + return 1; + } + if (bIndex === -1) { + return -1; + } + return aIndex - bIndex; + }); + const ConfigurationTags = ({ testConfig, clientVersion, @@ -25,28 +80,18 @@ const ConfigurationTags = ({ {clientVersion} )} - {Object.entries(testConfig || {}) - .filter(([k]) => k !== "BenchmarkRun" && k !== "GasLimit") - .map(([key, value]) => ( - - - {camelToTitleCase(key)}: - - {key === "GasLimit" ? ( - - {formatValue(Number(value), "gas")} - - ) : ( - - {String(formatLabel(`${value}`))} - - )} + {configEntries(testConfig).map(([key, value]) => ( + + + {configLabel(key)}: - ))} + {configValue(key, value)} + + ))}
); }; diff --git a/report/src/components/RunList.tsx b/report/src/components/RunList.tsx index 0265129b..6c485ba4 100644 --- a/report/src/components/RunList.tsx +++ b/report/src/components/RunList.tsx @@ -291,9 +291,13 @@ const RunList = ({ const statusCounts = groupBy(section.runs, "status"); const sortedRuns = isExpanded ? sortRuns(section.runs) : section.runs; const gasLimit = Number(section.runs?.[0]?.testConfig?.GasLimit); + const targetGasPerSecond = Number( + section.runs?.[0]?.testConfig?.TargetGPS, + ); const blockTimeMilliseconds = Number(section.runs?.[0]?.testConfig?.BlockTimeMilliseconds) || 2000; - const gasPerSecond = gasLimit / (blockTimeMilliseconds / 1000); + const gasPerSecond = + targetGasPerSecond || gasLimit / (blockTimeMilliseconds / 1000); return (
diff --git a/report/src/metricDefinitions.ts b/report/src/metricDefinitions.ts index 58085ebd..52a36b2b 100644 --- a/report/src/metricDefinitions.ts +++ b/report/src/metricDefinitions.ts @@ -192,6 +192,29 @@ export const CHART_CONFIG = { unit: "s", aliases: ["reth_op_rbuilder_state_root_calculation_duration"], }, + reth_base_builder_state_root_calculation_duration_quantile_0_5: { + type: "line", + title: "Builder State Root Calculation Duration p50", + description: "p50 time taken to calculate the state root", + unit: "s", + }, + reth_base_builder_state_root_calculation_duration_quantile_0_9: { + type: "line", + title: "Builder State Root Calculation Duration p90", + description: "p90 time taken to calculate the state root", + unit: "s", + }, + reth_base_builder_state_root_calculation_duration_quantile_0_99: { + type: "line", + title: "Builder State Root Calculation Duration p99", + description: "p99 time taken to calculate the state root", + unit: "s", + }, + reth_base_builder_state_root_time_per_gas_ratio_quantile_0_9: { + type: "line", + title: "Builder State Root Time per Gas p90", + description: "p90 state-root calculation time divided by gas processed", + }, reth_base_builder_sequencer_tx_duration_avg: { type: "line", title: "Builder Sequencer Tx Duration", @@ -224,6 +247,135 @@ export const CHART_CONFIG = { description: "Average gas headroom percentage across flashblocks", unit: "count", }, + reth_storage_providers_database_save_blocks_total_quantile_0_9: { + type: "line", + title: "Save Blocks Total p90", + description: "p90 total database save-blocks duration", + unit: "s", + }, + reth_storage_providers_database_save_blocks_block_count_last: { + type: "line", + title: "Save Blocks Block Count", + description: + "Number of blocks included in the most recent save-blocks operation", + unit: "blocks", + }, + reth_storage_providers_database_save_blocks_commit_sf_quantile_0_9: { + type: "line", + title: "Save Blocks Static File Commit p90", + description: "p90 static-file commit duration during save-blocks", + unit: "s", + }, + reth_storage_providers_database_save_blocks_commit_mdbx_quantile_0_9: { + type: "line", + title: "Save Blocks MDBX Commit p90", + description: "p90 MDBX commit duration during save-blocks", + unit: "s", + }, + reth_storage_providers_database_save_blocks_write_state_quantile_0_9: { + type: "line", + title: "Save Blocks Write State p90", + description: "p90 state write duration during save-blocks", + unit: "s", + }, + reth_storage_providers_database_save_blocks_write_hashed_state_quantile_0_9: { + type: "line", + title: "Save Blocks Write Hashed State p90", + description: "p90 hashed-state write duration during save-blocks", + unit: "s", + }, + reth_storage_providers_database_save_blocks_write_trie_updates_quantile_0_9: { + type: "line", + title: "Save Blocks Write Trie Updates p90", + description: "p90 trie-update write duration during save-blocks", + unit: "s", + }, + reth_storage_providers_database_save_blocks_sf_quantile_0_9: { + type: "line", + title: "Save Blocks Static Files p90", + description: "p90 static-file save-blocks duration", + unit: "s", + }, + reth_trie_leaves_added_quantile_0_9: { + type: "line", + title: "Trie Leaves Added p90", + description: "p90 trie leaves added", + unit: "count", + }, + reth_trie_branches_added_quantile_0_9: { + type: "line", + title: "Trie Branches Added p90", + description: "p90 trie branches added", + unit: "count", + }, + reth_tree_root_sparse_trie_total_duration_histogram_quantile_0_9: { + type: "line", + title: "Sparse Trie Total Duration p90", + description: "p90 sparse-trie total duration", + unit: "s", + aliases: ["reth_tree_root_sparse_trie_total_duration_histogram"], + }, + reth_tree_root_sparse_trie_final_update_duration_histogram_quantile_0_9: { + type: "line", + title: "Sparse Trie Final Update Duration p90", + description: "p90 sparse-trie final update duration", + unit: "s", + aliases: ["reth_tree_root_sparse_trie_final_update_duration_histogram"], + }, + reth_parallel_sparse_trie_subtrie_hash_update_latency_quantile_0_9: { + type: "line", + title: "Sparse Trie Subtrie Hash Update p90", + description: "p90 subtrie hash update latency", + unit: "s", + }, + reth_parallel_sparse_trie_subtrie_upper_hash_latency_quantile_0_9: { + type: "line", + title: "Sparse Trie Subtrie Upper Hash p90", + description: "p90 subtrie upper-hash latency", + unit: "s", + }, + reth_trie_proof_task_storage_worker_idle_time_seconds_quantile_0_9: { + type: "line", + title: "Trie Proof Storage Worker Idle p90", + description: "p90 trie-proof storage worker idle time", + unit: "s", + }, + reth_trie_proof_task_account_worker_idle_time_seconds_quantile_0_9: { + type: "line", + title: "Trie Proof Account Worker Idle p90", + description: "p90 trie-proof account worker idle time", + unit: "s", + }, + reth_trie_proof_task_blinded_storage_nodes_quantile_0_9: { + type: "line", + title: "Trie Proof Blinded Storage Nodes p90", + description: "p90 blinded storage nodes handled by trie proof tasks", + unit: "count", + }, + reth_trie_proof_task_blinded_account_nodes_quantile_0_9: { + type: "line", + title: "Trie Proof Blinded Account Nodes p90", + description: "p90 blinded account nodes handled by trie proof tasks", + unit: "count", + }, + reth_trie_cursor_overall_duration_quantile_0_9: { + type: "line", + title: "Trie Cursor Overall Duration p90", + description: "p90 trie cursor overall duration", + unit: "s", + }, + reth_trie_hashed_cursor_overall_duration_quantile_0_9: { + type: "line", + title: "Trie Hashed Cursor Overall Duration p90", + description: "p90 hashed trie cursor overall duration", + unit: "s", + }, + reth_db_freelist: { + type: "line", + title: "MDBX Freelist", + description: "MDBX freelist size", + unit: "count", + }, reth_sync_state_provider_total_storage_fetch_latency_avg: { type: "line", title: "Validator Storage Load Latency", @@ -268,6 +420,31 @@ const CHART_CONFIG_ORDER: (keyof typeof CHART_CONFIG)[] = [ "chain/storage/commits.50-percentile", "chain/snapshot/commits.50-percentile", "chain/triedb/commits.50-percentile", + "reth_base_builder_state_root_calculation_duration_quantile_0_5", + "reth_base_builder_state_root_calculation_duration_quantile_0_9", + "reth_base_builder_state_root_calculation_duration_quantile_0_99", + "reth_base_builder_state_root_time_per_gas_ratio_quantile_0_9", + "reth_storage_providers_database_save_blocks_total_quantile_0_9", + "reth_storage_providers_database_save_blocks_block_count_last", + "reth_storage_providers_database_save_blocks_commit_sf_quantile_0_9", + "reth_storage_providers_database_save_blocks_commit_mdbx_quantile_0_9", + "reth_storage_providers_database_save_blocks_write_state_quantile_0_9", + "reth_storage_providers_database_save_blocks_write_hashed_state_quantile_0_9", + "reth_storage_providers_database_save_blocks_write_trie_updates_quantile_0_9", + "reth_storage_providers_database_save_blocks_sf_quantile_0_9", + "reth_trie_leaves_added_quantile_0_9", + "reth_trie_branches_added_quantile_0_9", + "reth_tree_root_sparse_trie_total_duration_histogram_quantile_0_9", + "reth_tree_root_sparse_trie_final_update_duration_histogram_quantile_0_9", + "reth_parallel_sparse_trie_subtrie_hash_update_latency_quantile_0_9", + "reth_parallel_sparse_trie_subtrie_upper_hash_latency_quantile_0_9", + "reth_trie_proof_task_storage_worker_idle_time_seconds_quantile_0_9", + "reth_trie_proof_task_account_worker_idle_time_seconds_quantile_0_9", + "reth_trie_proof_task_blinded_storage_nodes_quantile_0_9", + "reth_trie_proof_task_blinded_account_nodes_quantile_0_9", + "reth_trie_cursor_overall_duration_quantile_0_9", + "reth_trie_hashed_cursor_overall_duration_quantile_0_9", + "reth_db_freelist", ]; export const SORTED_CHART_CONFIG: [string, ChartConfig][] = Object.entries( diff --git a/report/src/pages/RunIndex.tsx b/report/src/pages/RunIndex.tsx index 30caa7e2..f20d03e8 100644 --- a/report/src/pages/RunIndex.tsx +++ b/report/src/pages/RunIndex.tsx @@ -51,7 +51,12 @@ const RunIndexInner = ({ benchmarkRuns }: { benchmarkRuns: BenchmarkRuns }) => { } }); - const groups = groupBy(matchedRuns, "testConfig.GasLimit"); + const groups = groupBy(matchedRuns, (run) => { + if (run.testConfig.TargetGPS) { + return `target-gps-${run.testConfig.TargetGPS}`; + } + return `gas-limit-${run.testConfig.GasLimit}`; + }); // Build sections array with diffKeyStart const sections: { diff --git a/runner/clients/builder/metrics.go b/runner/clients/builder/metrics.go index d6f83366..d4bcd739 100644 --- a/runner/clients/builder/metrics.go +++ b/runner/clients/builder/metrics.go @@ -5,11 +5,17 @@ import ( "context" "fmt" "io" + "math" "net/http" + "sort" + "strconv" + "strings" + "unicode" "github.com/base/base-bench/runner/metrics" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" + io_prometheus_client "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" ) @@ -19,6 +25,7 @@ type metricsCollector struct { client *ethclient.Client metrics []metrics.BlockMetrics metricsPort int + prevMetrics map[string]*io_prometheus_client.Metric } func newMetricsCollector(log log.Logger, client *ethclient.Client, metricsPort int) metrics.Collector { @@ -27,6 +34,7 @@ func newMetricsCollector(log log.Logger, client *ethclient.Client, metricsPort i client: client, metricsPort: metricsPort, metrics: make([]metrics.BlockMetrics, 0), + prevMetrics: make(map[string]*io_prometheus_client.Metric), } } @@ -40,15 +48,48 @@ func (r *metricsCollector) GetMetrics() []metrics.BlockMetrics { func (r *metricsCollector) GetMetricTypes() map[string]bool { return map[string]bool{ - "reth_sync_execution_execution_duration": true, - "reth_sync_block_validation_state_root_duration": true, - "reth_op_rbuilder_block_built_success": true, - "reth_op_rbuilder_flashblock_count": true, - "reth_op_rbuilder_total_block_built_duration": true, - "reth_op_rbuilder_flashblock_build_duration": true, - "reth_op_rbuilder_state_root_calculation_duration": true, - "reth_op_rbuilder_sequencer_tx_duration": true, - "reth_op_rbuilder_payload_tx_simulation_duration": true, + "reth_sync_execution_execution_duration": true, + "reth_sync_block_validation_state_root_duration": true, + "reth_op_rbuilder_block_built_success": true, + "reth_op_rbuilder_flashblock_count": true, + "reth_op_rbuilder_total_block_built_duration": true, + "reth_op_rbuilder_flashblock_build_duration": true, + "reth_op_rbuilder_state_root_calculation_duration": true, + "reth_op_rbuilder_sequencer_tx_duration": true, + "reth_op_rbuilder_payload_tx_simulation_duration": true, + "reth_base_builder_block_built_success": true, + "reth_base_builder_flashblock_count": true, + "reth_base_builder_total_block_built_duration": true, + "reth_base_builder_flashblock_build_duration": true, + "reth_base_builder_state_root_calculation_duration": true, + "reth_base_builder_state_root_time_per_gas_ratio": true, + "reth_base_builder_sequencer_tx_duration": true, + "reth_base_builder_payload_transaction_simulation_duration": true, + "reth_base_builder_payload_tx_simulation_duration": true, + "reth_base_builder_tx_simulation_duration": true, + "reth_base_builder_payload_num_tx_gauge": true, + "reth_base_builder_flashblock_gas_headroom_pct": true, + "reth_storage_providers_database_save_blocks_total": true, + "reth_storage_providers_database_save_blocks_block_count_last": true, + "reth_storage_providers_database_save_blocks_commit_sf": true, + "reth_storage_providers_database_save_blocks_commit_mdbx": true, + "reth_storage_providers_database_save_blocks_write_state": true, + "reth_storage_providers_database_save_blocks_write_hashed_state": true, + "reth_storage_providers_database_save_blocks_write_trie_updates": true, + "reth_storage_providers_database_save_blocks_sf": true, + "reth_trie_leaves_added": true, + "reth_trie_branches_added": true, + "reth_tree_root_sparse_trie_total_duration_histogram": true, + "reth_tree_root_sparse_trie_final_update_duration_histogram": true, + "reth_parallel_sparse_trie_subtrie_hash_update_latency": true, + "reth_parallel_sparse_trie_subtrie_upper_hash_latency": true, + "reth_trie_proof_task_storage_worker_idle_time_seconds": true, + "reth_trie_proof_task_account_worker_idle_time_seconds": true, + "reth_trie_proof_task_blinded_storage_nodes": true, + "reth_trie_proof_task_blinded_account_nodes": true, + "reth_trie_cursor_overall_duration": true, + "reth_trie_hashed_cursor_overall_duration": true, + "reth_db_freelist": true, } } @@ -73,21 +114,139 @@ func (r *metricsCollector) Collect(ctx context.Context, m *metrics.BlockMetrics) } metricTypes := r.GetMetricTypes() + m.SetPreviousPrometheusMetrics(r.prevMetrics) for _, metric := range metrics { name := metric.GetName() if metricTypes[name] { metricVal := metric.GetMetric() - if len(metricVal) != 1 { - r.log.Warn("expected 1 metric, got %d for metric %s", len(metricVal), name) - } - err = m.UpdatePrometheusMetric(name, metricVal[0]) - if err != nil { - r.log.Warn("failed to add metric %s: %s", name, err) + for _, value := range metricVal { + metricName := prometheusMetricName(name, value) + addPrometheusMetric(r.log, m, metricName, value) + + if metricName != name && len(metricVal) == 1 { + addPrometheusMetric(r.log, m, name, value) + } } } } + r.prevMetrics = m.PreviousPrometheusMetrics() r.metrics = append(r.metrics, *m.Copy()) return nil } + +func addPrometheusMetric(log log.Logger, m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { + addHistogramQuantiles(m, name, metric, m.PreviousPrometheusMetric(name)) + + err := m.UpdatePrometheusMetric(name, metric) + if err != nil { + log.Warn("failed to add metric %s: %s", name, err) + } + addSummaryQuantiles(m, name, metric) +} + +func prometheusMetricName(name string, metric *io_prometheus_client.Metric) string { + labels := metric.GetLabel() + if len(labels) == 0 { + return name + } + + parts := make([]string, 0, len(labels)) + for _, label := range labels { + parts = append(parts, sanitizeMetricPart(label.GetName())+"_"+sanitizeMetricPart(label.GetValue())) + } + sort.Strings(parts) + return name + "_" + strings.Join(parts, "_") +} + +func addSummaryQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { + if metric.Summary == nil { + return + } + + for _, quantile := range metric.Summary.GetQuantile() { + m.AddExecutionMetric(name+"_quantile_"+formatQuantile(quantile.GetQuantile()), quantile.GetValue()) + } +} + +func addHistogramQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric, prevMetric *io_prometheus_client.Metric) { + if metric.Histogram == nil { + return + } + + var prevHistogram *io_prometheus_client.Histogram + if prevMetric != nil { + prevHistogram = prevMetric.Histogram + } + + for _, quantile := range []float64{0.5, 0.9, 0.99} { + value, ok := histogramQuantile(quantile, metric.Histogram, prevHistogram) + if ok { + m.AddExecutionMetric(name+"_quantile_"+formatQuantile(quantile), value) + } + } +} + +func histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogram, prevHistogram *io_prometheus_client.Histogram) (float64, bool) { + if histogram == nil || histogram.SampleCount == nil || len(histogram.Bucket) == 0 { + return 0, false + } + + prevCount := uint64(0) + if prevHistogram != nil && prevHistogram.SampleCount != nil { + prevCount = *prevHistogram.SampleCount + } + count := deltaUint64(*histogram.SampleCount, prevCount) + if count == 0 { + return 0, false + } + + rank := quantile * float64(count) + lastFiniteUpperBound := 0.0 + hasFiniteUpperBound := false + for i, bucket := range histogram.Bucket { + if bucket.CumulativeCount == nil || bucket.UpperBound == nil { + continue + } + if !math.IsInf(*bucket.UpperBound, 0) { + lastFiniteUpperBound = *bucket.UpperBound + hasFiniteUpperBound = true + } + + prevBucketCount := uint64(0) + if prevHistogram != nil && i < len(prevHistogram.Bucket) && prevHistogram.Bucket[i].CumulativeCount != nil { + prevBucketCount = *prevHistogram.Bucket[i].CumulativeCount + } + + bucketCount := deltaUint64(*bucket.CumulativeCount, prevBucketCount) + if float64(bucketCount) >= rank { + if math.IsInf(*bucket.UpperBound, 0) { + return lastFiniteUpperBound, hasFiniteUpperBound + } + return *bucket.UpperBound, true + } + } + + return 0, false +} + +func deltaUint64(current uint64, previous uint64) uint64 { + if current < previous { + return current + } + return current - previous +} + +func formatQuantile(quantile float64) string { + return strings.ReplaceAll(strconv.FormatFloat(quantile, 'f', -1, 64), ".", "_") +} + +func sanitizeMetricPart(value string) string { + return strings.Map(func(r rune) rune { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + return r + } + return '_' + }, value) +} diff --git a/runner/clients/builder/metrics_test.go b/runner/clients/builder/metrics_test.go new file mode 100644 index 00000000..c5d462e4 --- /dev/null +++ b/runner/clients/builder/metrics_test.go @@ -0,0 +1,98 @@ +package builder + +import ( + "math" + "testing" + + io_prometheus_client "github.com/prometheus/client_model/go" +) + +func TestPrometheusMetricNameSortsAndSanitizesLabels(t *testing.T) { + metric := &io_prometheus_client.Metric{ + Label: []*io_prometheus_client.LabelPair{ + {Name: stringPtr("type"), Value: stringPtr("account.worker")}, + {Name: stringPtr("configname"), Value: stringPtr("mainnet/snapshot")}, + }, + } + + got := prometheusMetricName("reth_example_metric", metric) + want := "reth_example_metric_configname_mainnet_snapshot_type_account_worker" + if got != want { + t.Fatalf("prometheusMetricName() = %q, want %q", got, want) + } +} + +func TestHistogramQuantileUsesIntervalBuckets(t *testing.T) { + prev := histogramMetric( + 10, + bucket(1, 5), + bucket(2, 9), + bucket(3, 10), + ) + current := histogramMetric( + 20, + bucket(1, 8), + bucket(2, 17), + bucket(3, 20), + ) + + got, ok := histogramQuantile(0.5, current.Histogram, prev.Histogram) + if !ok { + t.Fatal("histogramQuantile() did not return a value") + } + if got != 2 { + t.Fatalf("histogramQuantile(0.5) = %f, want 2", got) + } + + got, ok = histogramQuantile(0.9, current.Histogram, prev.Histogram) + if !ok { + t.Fatal("histogramQuantile() did not return a value") + } + if got != 3 { + t.Fatalf("histogramQuantile(0.9) = %f, want 3", got) + } +} + +func TestHistogramQuantileAvoidsInfiniteUpperBound(t *testing.T) { + current := histogramMetric( + 10, + bucket(1, 5), + bucket(math.Inf(1), 10), + ) + + got, ok := histogramQuantile(0.9, current.Histogram, nil) + if !ok { + t.Fatal("histogramQuantile() did not return a value") + } + if got != 1 { + t.Fatalf("histogramQuantile(0.9) = %f, want 1", got) + } +} + +func histogramMetric(count uint64, buckets ...*io_prometheus_client.Bucket) *io_prometheus_client.Metric { + return &io_prometheus_client.Metric{ + Histogram: &io_prometheus_client.Histogram{ + SampleCount: uint64Ptr(count), + Bucket: buckets, + }, + } +} + +func bucket(upperBound float64, count uint64) *io_prometheus_client.Bucket { + return &io_prometheus_client.Bucket{ + UpperBound: float64Ptr(upperBound), + CumulativeCount: uint64Ptr(count), + } +} + +func stringPtr(value string) *string { + return &value +} + +func float64Ptr(value float64) *float64 { + return &value +} + +func uint64Ptr(value uint64) *uint64 { + return &value +} diff --git a/runner/metrics/metrics_interface.go b/runner/metrics/metrics_interface.go index 00c0811f..350dd3f9 100644 --- a/runner/metrics/metrics_interface.go +++ b/runner/metrics/metrics_interface.go @@ -51,6 +51,21 @@ func (m *BlockMetrics) Copy() *BlockMetrics { } } +func (m *BlockMetrics) SetPreviousPrometheusMetrics(prev map[string]*io_prometheus_client.Metric) { + m.prevMetrics = make(map[string]*io_prometheus_client.Metric, len(prev)) + maps.Copy(m.prevMetrics, prev) +} + +func (m *BlockMetrics) PreviousPrometheusMetrics() map[string]*io_prometheus_client.Metric { + prevMetrics := make(map[string]*io_prometheus_client.Metric, len(m.prevMetrics)) + maps.Copy(prevMetrics, m.prevMetrics) + return prevMetrics +} + +func (m *BlockMetrics) PreviousPrometheusMetric(name string) *io_prometheus_client.Metric { + return m.prevMetrics[name] +} + func (m *BlockMetrics) UpdatePrometheusMetric(name string, value *io_prometheus_client.Metric) error { if value.Histogram != nil { // get the average change in sum divided by the average change in count From 8c1dbb79d7656dc1549d710ec25e2aabe8a91874 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 27 May 2026 00:24:12 -0500 Subject: [PATCH 11/15] fix load test benchmark lifecycle --- report/src/components/ChartGrid.tsx | 6 +- runner/network/sequencer_benchmark.go | 142 +++++++++--------- runner/payload/loadtest/load_test_worker.go | 77 ++++++---- .../payload/loadtest/load_test_worker_test.go | 38 +++++ 4 files changed, 160 insertions(+), 103 deletions(-) diff --git a/report/src/components/ChartGrid.tsx b/report/src/components/ChartGrid.tsx index d339d146..fe4596e5 100644 --- a/report/src/components/ChartGrid.tsx +++ b/report/src/components/ChartGrid.tsx @@ -27,13 +27,13 @@ function resolveMetricKey( const metricPrefix = quantileSuffix ? key.slice(0, -quantileSuffix.length) : key; - const labeledMetricKey = metricKeys.find( + const labeledMetricKeys = metricKeys.filter( (metricKey) => metricKey.startsWith(`${metricPrefix}_`) && (!quantileSuffix || metricKey.endsWith(quantileSuffix)), ); - if (labeledMetricKey) { - return labeledMetricKey; + if (labeledMetricKeys.length === 1) { + return labeledMetricKeys[0]; } } return primaryKey; diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index 45c0fd82..5562df2c 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -27,6 +27,35 @@ import ( const gracefulWorkerShutdownTimeout = 90 * time.Second +type benchmarkRunController struct { + maxBlocks int + completion payloadworker.CompletionWorker +} + +func newBenchmarkRunController(transactionWorker payloadworker.Worker, params benchtypes.RunParams) benchmarkRunController { + completion, ok := transactionWorker.(payloadworker.CompletionWorker) + if ok { + return benchmarkRunController{completion: completion} + } + return benchmarkRunController{maxBlocks: params.NumBlocks} +} + +func (c benchmarkRunController) shouldStop(nextBlockIndex uint64) (bool, error) { + if c.completion != nil { + select { + case <-c.completion.Done(): + return true, c.completion.Err() + default: + return false, nil + } + } + return int(nextBlockIndex) > c.maxBlocks, nil +} + +func (c benchmarkRunController) usesWorkerCompletion() bool { + return c.completion != nil +} + type sequencerBenchmark struct { log log.Logger sequencerClient types.ExecutionClient @@ -241,58 +270,45 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. payloads = append(payloads, *lastSetupPayload) pendingTxs := 0 - completionWorker, runUntilWorkerDone := transactionWorker.(payloadworker.CompletionWorker) - if runUntilWorkerDone { + runController := newBenchmarkRunController(transactionWorker, params) + if runController.usesWorkerCompletion() { nb.log.Info("Running benchmark blocks until payload worker completes") - blockIndex := uint64(1) - runUntilDoneLoop: - for { - select { - case <-completionWorker.Done(): - if err := completionWorker.Err(); err != nil { - errChan <- errors.Wrap(err, "payload worker failed") - return - } - nb.log.Info("Payload worker completed", "blocks", blockIndex-1) - break runUntilDoneLoop - default: - } + } - payload, updatedPendingTxs, err := nb.proposeBenchmarkBlock( - benchmarkCtx, - transactionWorker, - consensusClient, - metricsCollector, - blockIndex, - pendingTxs, - ) - if err != nil { - errChan <- err - return - } - pendingTxs = updatedPendingTxs - payloads = append(payloads, *payload) - blockIndex++ + blockIndex := uint64(1) + for { + stop, err := runController.shouldStop(blockIndex) + if err != nil { + errChan <- errors.Wrap(err, "payload worker failed") + return } - } else { - // run for a few blocks - for i := 0; i < params.NumBlocks; i++ { - payload, updatedPendingTxs, err := nb.proposeBenchmarkBlock( - benchmarkCtx, - transactionWorker, - consensusClient, - metricsCollector, - uint64(i)+1, - pendingTxs, - ) - if err != nil { - errChan <- err - return + if stop { + if runController.usesWorkerCompletion() { + nb.log.Info("Payload worker completed", "blocks", blockIndex-1) } - pendingTxs = updatedPendingTxs - payloads = append(payloads, *payload) + break + } + + payload, updatedPendingTxs, err := nb.proposeBlock( + benchmarkCtx, + transactionWorker, + consensusClient, + metricsCollector, + blockIndex, + pendingTxs, + false, + true, + ) + if err != nil { + errChan <- err + return } + pendingTxs = updatedPendingTxs + payloads = append(payloads, *payload) + blockIndex++ + } + if !runController.usesWorkerCompletion() { if err := nb.settleGracefulWorkerShutdown(benchmarkCtx, transactionWorker, consensusClient, pendingTxs); err != nil { errChan <- err return @@ -325,13 +341,15 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. } } -func (nb *sequencerBenchmark) proposeBenchmarkBlock( +func (nb *sequencerBenchmark) proposeBlock( ctx context.Context, transactionWorker payloadworker.Worker, consensusClient *consensus.SequencerConsensusClient, metricsCollector metrics.Collector, blockIndex uint64, pendingTxs int, + isSetupPayload bool, + collectMetrics bool, ) (*engine.ExecutableData, int, error) { blockMetrics := metrics.NewBlockMetrics() blockMetrics.SetBlockNumber(blockIndex) @@ -342,7 +360,7 @@ func (nb *sequencerBenchmark) proposeBenchmarkBlock( return nil, pendingTxs, err } - payload, err := consensusClient.Propose(ctx, blockMetrics, false) + payload, err := consensusClient.Propose(ctx, blockMetrics, isSetupPayload) if err != nil { return nil, pendingTxs, err } @@ -366,8 +384,10 @@ func (nb *sequencerBenchmark) proposeBenchmarkBlock( time.Sleep(nb.config.Params.BlockTime) } - if err := metricsCollector.Collect(ctx, blockMetrics); err != nil { - nb.log.Error("Failed to collect metrics", "error", err) + if collectMetrics { + if err := metricsCollector.Collect(ctx, blockMetrics); err != nil { + nb.log.Error("Failed to collect metrics", "error", err) + } } return payload, updatedPendingTxs, nil @@ -403,13 +423,9 @@ func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( default: } - blockMetrics := metrics.NewBlockMetrics() - txsSent, err := transactionWorker.SendTxs(ctx, pendingTxs) - if err != nil { - return errors.Wrap(err, "failed to collect settlement transactions") - } - - payload, err := consensusClient.Propose(ctx, blockMetrics, true) + var payload *engine.ExecutableData + var err error + payload, pendingTxs, err = nb.proposeBlock(ctx, transactionWorker, consensusClient, nil, uint64(settlementBlock+1), pendingTxs, true, false) if err != nil { return errors.Wrap(err, "failed to propose settlement block") } @@ -417,19 +433,7 @@ func (nb *sequencerBenchmark) settleGracefulWorkerShutdown( return errors.New("received nil settlement payload from consensus client") } - userTxsIncluded := len(payload.Transactions) - 1 - if userTxsIncluded < 0 { - userTxsIncluded = 0 - } - pendingTxs = pendingTxs + txsSent - userTxsIncluded - if pendingTxs < 0 { - pendingTxs = 0 - } - settlementBlock++ - if !nb.config.Params.UseBaseConsensusTiming() { - time.Sleep(nb.config.Params.BlockTime) - } } } diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index ecb557cb..b59bb333 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -42,6 +42,7 @@ type loadTestPayloadWorker struct { mempool *mempool.StaticWorkloadMempool cmd *exec.Cmd done chan struct{} + startOnce sync.Once shutdownOnce sync.Once waitErrMu sync.Mutex waitErr error @@ -80,6 +81,7 @@ func NewLoadTestPayloadWorker( targetGPS: params.TargetGPS, params: definition, mempool: mp, + done: make(chan struct{}), sourceConfigPath: sourceConfigPath, outputPath: outputPath, } @@ -98,33 +100,42 @@ func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { } w.renderedConfigPath = configPath - w.log.Info("Starting load test", "binary", w.loadTestBin, "config", configPath) + w.log.Info("Prepared load test", "binary", w.loadTestBin, "config", configPath) + return nil +} - cmd := exec.CommandContext(ctx, w.loadTestBin, configPath) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout - cmd.Env = append(os.Environ(), fmt.Sprintf("FUNDER_KEY=%s", w.prefundSK)) - if w.outputPath != "" { - if err := os.MkdirAll(filepath.Dir(w.outputPath), 0755); err != nil { - return errors.Wrap(err, "failed to create load-test output directory") +func (w *loadTestPayloadWorker) start(ctx context.Context) error { + w.startOnce.Do(func() { + if w.renderedConfigPath == "" { + w.finish(errors.New("load-test config has not been prepared")) + return } - cmd.Env = append(cmd.Env, fmt.Sprintf("LOAD_TEST_OUTPUT=%s", w.outputPath)) - } - if err := cmd.Start(); err != nil { - return errors.Wrap(err, "failed to start load test binary") - } - w.cmd = cmd - w.done = make(chan struct{}) - go func() { - err := cmd.Wait() - w.waitErrMu.Lock() - w.waitErr = err - w.waitErrMu.Unlock() - close(w.done) - }() + w.log.Info("Starting load test", "binary", w.loadTestBin, "config", w.renderedConfigPath) - return nil + cmd := exec.CommandContext(ctx, w.loadTestBin, w.renderedConfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + cmd.Env = append(os.Environ(), fmt.Sprintf("FUNDER_KEY=%s", w.prefundSK)) + if w.outputPath != "" { + if err := os.MkdirAll(filepath.Dir(w.outputPath), 0755); err != nil { + w.finish(errors.Wrap(err, "failed to create load-test output directory")) + return + } + cmd.Env = append(cmd.Env, fmt.Sprintf("LOAD_TEST_OUTPUT=%s", w.outputPath)) + } + + if err := cmd.Start(); err != nil { + w.finish(errors.Wrap(err, "failed to start load test binary")) + return + } + w.cmd = cmd + go func() { + w.finish(cmd.Wait()) + }() + }) + + return w.Err() } func (w *loadTestPayloadWorker) BeginGracefulShutdown(ctx context.Context) error { @@ -156,13 +167,7 @@ func (w *loadTestPayloadWorker) BeginGracefulShutdown(ctx context.Context) error } func (w *loadTestPayloadWorker) Done() <-chan struct{} { - if w.done != nil { - return w.done - } - - done := make(chan struct{}) - close(done) - return done + return w.done } func (w *loadTestPayloadWorker) Err() error { @@ -171,6 +176,13 @@ func (w *loadTestPayloadWorker) Err() error { return w.waitErr } +func (w *loadTestPayloadWorker) finish(err error) { + w.waitErrMu.Lock() + w.waitErr = err + w.waitErrMu.Unlock() + close(w.done) +} + func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { if w.cmd != nil && w.cmd.Process != nil { if err := w.BeginGracefulShutdown(ctx); err != nil { @@ -201,7 +213,10 @@ func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { return nil } -func (w *loadTestPayloadWorker) SendTxs(_ context.Context, _ int) (int, error) { +func (w *loadTestPayloadWorker) SendTxs(ctx context.Context, _ int) (int, error) { + if err := w.start(ctx); err != nil { + return 0, err + } return 0, nil } diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go index 7bef24bb..ec59b8e3 100644 --- a/runner/payload/loadtest/load_test_worker_test.go +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -1,10 +1,12 @@ package loadtest import ( + "context" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -139,6 +141,42 @@ transactions: require.Contains(t, output, "duration: \"60s\"") } +func TestSetupPreparesConfigWithoutStartingProcess(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "load-test.yaml") + err := os.WriteFile(configPath, []byte(` +transaction_submission_rpcs: + - "http://standalone-submitter.invalid" +query_rpc: "http://standalone-query.invalid" +duration: "60s" +transactions: + - weight: 100 + type: transfer +`), 0644) + require.NoError(t, err) + + worker := &loadTestPayloadWorker{ + log: log.New(), + elRPCURL: "http://sequencer.example", + sourceConfigPath: configPath, + done: make(chan struct{}), + } + t.Cleanup(func() { + if worker.renderedConfigPath != "" { + require.NoError(t, os.Remove(worker.renderedConfigPath)) + } + }) + + require.NoError(t, worker.Setup(context.Background())) + require.NotEmpty(t, worker.renderedConfigPath) + require.Nil(t, worker.cmd) + + select { + case <-worker.Done(): + t.Fatal("load-test worker should not be done before it starts") + default: + } +} + func TestResolveConfigFilePath(t *testing.T) { resolved, err := resolveConfigFilePath("/tmp/configs/benchmark.yml", "load-tests/mainnet.yaml") require.NoError(t, err) From 49ac1f1ea805a51c744d96f6d41c51c239aede3f Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 27 May 2026 00:26:40 -0500 Subject: [PATCH 12/15] simplify proxy pending tx draining --- runner/clients/common/proxy/proxy.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/runner/clients/common/proxy/proxy.go b/runner/clients/common/proxy/proxy.go index bf3f8147..b3697198 100644 --- a/runner/clients/common/proxy/proxy.go +++ b/runner/clients/common/proxy/proxy.go @@ -80,28 +80,11 @@ func (p *ProxyServer) Run(ctx context.Context) error { return nil } -func (p *ProxyServer) PendingTxs() []*ethTypes.Transaction { - p.mu.Lock() - defer p.mu.Unlock() - - txs := make([]*ethTypes.Transaction, len(p.pendingTxs)) - copy(txs, p.pendingTxs) - return txs -} - -func (p *ProxyServer) ClearPendingTxs() { - p.mu.Lock() - defer p.mu.Unlock() - - p.pendingTxs = make([]*ethTypes.Transaction, 0) -} - func (p *ProxyServer) DrainPendingTxs() []*ethTypes.Transaction { p.mu.Lock() defer p.mu.Unlock() - txs := make([]*ethTypes.Transaction, len(p.pendingTxs)) - copy(txs, p.pendingTxs) + txs := p.pendingTxs p.pendingTxs = make([]*ethTypes.Transaction, 0) return txs } From 80ff4bc5336892faa3fb5dfcee3fd038e4c51141 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 27 May 2026 14:37:56 -0500 Subject: [PATCH 13/15] scope builder metric helpers to collector --- runner/clients/builder/metrics.go | 40 +++++++++++++------------- runner/clients/builder/metrics_test.go | 11 ++++--- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/runner/clients/builder/metrics.go b/runner/clients/builder/metrics.go index d4bcd739..0bed149d 100644 --- a/runner/clients/builder/metrics.go +++ b/runner/clients/builder/metrics.go @@ -121,11 +121,11 @@ func (r *metricsCollector) Collect(ctx context.Context, m *metrics.BlockMetrics) if metricTypes[name] { metricVal := metric.GetMetric() for _, value := range metricVal { - metricName := prometheusMetricName(name, value) - addPrometheusMetric(r.log, m, metricName, value) + metricName := r.prometheusMetricName(name, value) + r.addPrometheusMetric(m, metricName, value) if metricName != name && len(metricVal) == 1 { - addPrometheusMetric(r.log, m, name, value) + r.addPrometheusMetric(m, name, value) } } } @@ -136,17 +136,17 @@ func (r *metricsCollector) Collect(ctx context.Context, m *metrics.BlockMetrics) return nil } -func addPrometheusMetric(log log.Logger, m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { - addHistogramQuantiles(m, name, metric, m.PreviousPrometheusMetric(name)) +func (r *metricsCollector) addPrometheusMetric(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { + r.addHistogramQuantiles(m, name, metric, m.PreviousPrometheusMetric(name)) err := m.UpdatePrometheusMetric(name, metric) if err != nil { - log.Warn("failed to add metric %s: %s", name, err) + r.log.Warn("failed to add metric %s: %s", name, err) } - addSummaryQuantiles(m, name, metric) + r.addSummaryQuantiles(m, name, metric) } -func prometheusMetricName(name string, metric *io_prometheus_client.Metric) string { +func (r *metricsCollector) prometheusMetricName(name string, metric *io_prometheus_client.Metric) string { labels := metric.GetLabel() if len(labels) == 0 { return name @@ -154,23 +154,23 @@ func prometheusMetricName(name string, metric *io_prometheus_client.Metric) stri parts := make([]string, 0, len(labels)) for _, label := range labels { - parts = append(parts, sanitizeMetricPart(label.GetName())+"_"+sanitizeMetricPart(label.GetValue())) + parts = append(parts, r.sanitizeMetricPart(label.GetName())+"_"+r.sanitizeMetricPart(label.GetValue())) } sort.Strings(parts) return name + "_" + strings.Join(parts, "_") } -func addSummaryQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { +func (r *metricsCollector) addSummaryQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { if metric.Summary == nil { return } for _, quantile := range metric.Summary.GetQuantile() { - m.AddExecutionMetric(name+"_quantile_"+formatQuantile(quantile.GetQuantile()), quantile.GetValue()) + m.AddExecutionMetric(name+"_quantile_"+r.formatQuantile(quantile.GetQuantile()), quantile.GetValue()) } } -func addHistogramQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric, prevMetric *io_prometheus_client.Metric) { +func (r *metricsCollector) addHistogramQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric, prevMetric *io_prometheus_client.Metric) { if metric.Histogram == nil { return } @@ -181,14 +181,14 @@ func addHistogramQuantiles(m *metrics.BlockMetrics, name string, metric *io_prom } for _, quantile := range []float64{0.5, 0.9, 0.99} { - value, ok := histogramQuantile(quantile, metric.Histogram, prevHistogram) + value, ok := r.histogramQuantile(quantile, metric.Histogram, prevHistogram) if ok { - m.AddExecutionMetric(name+"_quantile_"+formatQuantile(quantile), value) + m.AddExecutionMetric(name+"_quantile_"+r.formatQuantile(quantile), value) } } } -func histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogram, prevHistogram *io_prometheus_client.Histogram) (float64, bool) { +func (r *metricsCollector) histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogram, prevHistogram *io_prometheus_client.Histogram) (float64, bool) { if histogram == nil || histogram.SampleCount == nil || len(histogram.Bucket) == 0 { return 0, false } @@ -197,7 +197,7 @@ func histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogr if prevHistogram != nil && prevHistogram.SampleCount != nil { prevCount = *prevHistogram.SampleCount } - count := deltaUint64(*histogram.SampleCount, prevCount) + count := r.deltaUint64(*histogram.SampleCount, prevCount) if count == 0 { return 0, false } @@ -219,7 +219,7 @@ func histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogr prevBucketCount = *prevHistogram.Bucket[i].CumulativeCount } - bucketCount := deltaUint64(*bucket.CumulativeCount, prevBucketCount) + bucketCount := r.deltaUint64(*bucket.CumulativeCount, prevBucketCount) if float64(bucketCount) >= rank { if math.IsInf(*bucket.UpperBound, 0) { return lastFiniteUpperBound, hasFiniteUpperBound @@ -231,18 +231,18 @@ func histogramQuantile(quantile float64, histogram *io_prometheus_client.Histogr return 0, false } -func deltaUint64(current uint64, previous uint64) uint64 { +func (r *metricsCollector) deltaUint64(current uint64, previous uint64) uint64 { if current < previous { return current } return current - previous } -func formatQuantile(quantile float64) string { +func (r *metricsCollector) formatQuantile(quantile float64) string { return strings.ReplaceAll(strconv.FormatFloat(quantile, 'f', -1, 64), ".", "_") } -func sanitizeMetricPart(value string) string { +func (r *metricsCollector) sanitizeMetricPart(value string) string { return strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return r diff --git a/runner/clients/builder/metrics_test.go b/runner/clients/builder/metrics_test.go index c5d462e4..af78a911 100644 --- a/runner/clients/builder/metrics_test.go +++ b/runner/clients/builder/metrics_test.go @@ -8,6 +8,7 @@ import ( ) func TestPrometheusMetricNameSortsAndSanitizesLabels(t *testing.T) { + collector := &metricsCollector{} metric := &io_prometheus_client.Metric{ Label: []*io_prometheus_client.LabelPair{ {Name: stringPtr("type"), Value: stringPtr("account.worker")}, @@ -15,7 +16,7 @@ func TestPrometheusMetricNameSortsAndSanitizesLabels(t *testing.T) { }, } - got := prometheusMetricName("reth_example_metric", metric) + got := collector.prometheusMetricName("reth_example_metric", metric) want := "reth_example_metric_configname_mainnet_snapshot_type_account_worker" if got != want { t.Fatalf("prometheusMetricName() = %q, want %q", got, want) @@ -23,6 +24,7 @@ func TestPrometheusMetricNameSortsAndSanitizesLabels(t *testing.T) { } func TestHistogramQuantileUsesIntervalBuckets(t *testing.T) { + collector := &metricsCollector{} prev := histogramMetric( 10, bucket(1, 5), @@ -36,7 +38,7 @@ func TestHistogramQuantileUsesIntervalBuckets(t *testing.T) { bucket(3, 20), ) - got, ok := histogramQuantile(0.5, current.Histogram, prev.Histogram) + got, ok := collector.histogramQuantile(0.5, current.Histogram, prev.Histogram) if !ok { t.Fatal("histogramQuantile() did not return a value") } @@ -44,7 +46,7 @@ func TestHistogramQuantileUsesIntervalBuckets(t *testing.T) { t.Fatalf("histogramQuantile(0.5) = %f, want 2", got) } - got, ok = histogramQuantile(0.9, current.Histogram, prev.Histogram) + got, ok = collector.histogramQuantile(0.9, current.Histogram, prev.Histogram) if !ok { t.Fatal("histogramQuantile() did not return a value") } @@ -54,13 +56,14 @@ func TestHistogramQuantileUsesIntervalBuckets(t *testing.T) { } func TestHistogramQuantileAvoidsInfiniteUpperBound(t *testing.T) { + collector := &metricsCollector{} current := histogramMetric( 10, bucket(1, 5), bucket(math.Inf(1), 10), ) - got, ok := histogramQuantile(0.9, current.Histogram, nil) + got, ok := collector.histogramQuantile(0.9, current.Histogram, nil) if !ok { t.Fatal("histogramQuantile() did not return a value") } From e63db2d1166685ca22cbac4a6205eb18bba6056a Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 27 May 2026 20:07:34 -0500 Subject: [PATCH 14/15] Preserve builder Prometheus sample diagnostics --- runner/clients/builder/metrics.go | 183 ++++++++++++++++++++++++- runner/clients/builder/metrics_test.go | 70 ++++++++++ runner/metrics/metrics_interface.go | 64 +++++++-- 3 files changed, 301 insertions(+), 16 deletions(-) diff --git a/runner/clients/builder/metrics.go b/runner/clients/builder/metrics.go index 0bed149d..5c0152ae 100644 --- a/runner/clients/builder/metrics.go +++ b/runner/clients/builder/metrics.go @@ -67,8 +67,62 @@ func (r *metricsCollector) GetMetricTypes() map[string]bool { "reth_base_builder_payload_transaction_simulation_duration": true, "reth_base_builder_payload_tx_simulation_duration": true, "reth_base_builder_tx_simulation_duration": true, + "reth_base_builder_transaction_pool_fetch_duration": true, + "reth_base_builder_transaction_pool_fetch_gauge": true, + "reth_base_builder_state_transition_merge_duration": true, + "reth_base_builder_state_transition_merge_gauge": true, + "reth_base_builder_payload_num_tx_considered": true, + "reth_base_builder_payload_num_tx_considered_gauge": true, + "reth_base_builder_payload_num_tx": true, "reth_base_builder_payload_num_tx_gauge": true, + "reth_base_builder_payload_num_tx_simulated": true, + "reth_base_builder_payload_num_tx_simulated_gauge": true, + "reth_base_builder_payload_num_tx_simulated_success": true, + "reth_base_builder_payload_num_tx_simulated_success_gauge": true, + "reth_base_builder_payload_num_tx_simulated_fail": true, + "reth_base_builder_payload_num_tx_simulated_fail_gauge": true, + "reth_base_builder_payload_reverted_tx_gas_used": true, + "reth_base_builder_reverted_tx_gas_used": true, + "reth_base_builder_successful_tx_gas_used": true, + "reth_base_builder_tx_accounts_modified": true, + "reth_base_builder_tx_storage_slots_modified": true, + "reth_base_builder_rejection_cache_hits": true, + "reth_base_builder_rejection_cache_insertions": true, + "reth_base_builder_rejection_cache_size": true, + "reth_base_builder_metering_data_pending_skip": true, + "reth_base_builder_gas_limit_exceeded_total": true, + "reth_base_builder_tx_da_size_exceeded_total": true, + "reth_base_builder_block_da_size_exceeded_total": true, + "reth_base_builder_da_footprint_exceeded_total": true, + "reth_base_builder_block_uncompressed_size_exceeded_total": true, + "reth_base_builder_block_uncompressed_size": true, + "reth_base_builder_resource_limit_would_reject_total": true, + "reth_base_builder_tx_execution_time_exceeded_total": true, + "reth_base_builder_flashblock_execution_time_exceeded_total": true, + "reth_base_builder_block_state_root_gas_exceeded_total": true, + "reth_base_builder_flashblock_txs_considered": true, + "reth_base_builder_flashblock_txs_included": true, + "reth_base_builder_flashblock_txs_rejected": true, + "reth_base_builder_flashblock_selection_total": true, + "reth_base_builder_flashblock_rejections_total": true, + "reth_base_builder_flashblock_gas_headroom": true, "reth_base_builder_flashblock_gas_headroom_pct": true, + "reth_base_builder_flashblock_da_bytes_used": true, + "reth_base_builder_flashblock_da_headroom_bytes": true, + "reth_base_builder_flashblock_execution_time_used_us": true, + "reth_base_builder_flashblock_execution_time_headroom_us": true, + "reth_base_builder_flashblock_state_root_gas_used": true, + "reth_base_builder_flashblock_state_root_gas_headroom": true, + "reth_base_builder_flashblock_byte_size_histogram": true, + "reth_base_builder_flashblock_num_tx_histogram": true, + "reth_base_builder_payload_byte_size": true, + "reth_base_builder_payload_byte_size_gauge": true, + "reth_base_builder_tx_byte_size": true, + "reth_base_builder_block_state_root_gas": true, + "reth_base_builder_flashblocks_time_drift": true, + "reth_base_builder_first_flashblock_time_offset": true, + "reth_base_builder_reduced_flashblocks_number": true, + "reth_base_builder_missing_flashblocks_count": true, "reth_storage_providers_database_save_blocks_total": true, "reth_storage_providers_database_save_blocks_block_count_last": true, "reth_storage_providers_database_save_blocks_commit_sf": true, @@ -122,10 +176,10 @@ func (r *metricsCollector) Collect(ctx context.Context, m *metrics.BlockMetrics) metricVal := metric.GetMetric() for _, value := range metricVal { metricName := r.prometheusMetricName(name, value) - r.addPrometheusMetric(m, metricName, value) + r.addPrometheusMetric(m, name, metricName, value, true) if metricName != name && len(metricVal) == 1 { - r.addPrometheusMetric(m, name, value) + r.addPrometheusMetric(m, name, name, value, false) } } } @@ -136,8 +190,12 @@ func (r *metricsCollector) Collect(ctx context.Context, m *metrics.BlockMetrics) return nil } -func (r *metricsCollector) addPrometheusMetric(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { - r.addHistogramQuantiles(m, name, metric, m.PreviousPrometheusMetric(name)) +func (r *metricsCollector) addPrometheusMetric(m *metrics.BlockMetrics, rawName string, name string, metric *io_prometheus_client.Metric, recordSample bool) { + prevMetric := m.PreviousPrometheusMetric(name) + r.addHistogramQuantiles(m, name, metric, prevMetric) + if recordSample { + m.AddPrometheusMetricSample(r.prometheusMetricSample(rawName, name, metric, prevMetric)) + } err := m.UpdatePrometheusMetric(name, metric) if err != nil { @@ -146,6 +204,95 @@ func (r *metricsCollector) addPrometheusMetric(m *metrics.BlockMetrics, name str r.addSummaryQuantiles(m, name, metric) } +func (r *metricsCollector) prometheusMetricSample(rawName string, key string, metric *io_prometheus_client.Metric, prevMetric *io_prometheus_client.Metric) metrics.PrometheusMetricSample { + sample := metrics.PrometheusMetricSample{ + Name: rawName, + Key: key, + Labels: r.prometheusLabels(metric), + } + + if metric.Gauge != nil { + sample.Type = "gauge" + if metric.Gauge.Value != nil && !math.IsNaN(*metric.Gauge.Value) { + sample.Value = prometheusFloat64Ptr(*metric.Gauge.Value) + } + return sample + } + + if metric.Counter != nil { + sample.Type = "counter" + if metric.Counter.Value != nil && !math.IsNaN(*metric.Counter.Value) { + value := *metric.Counter.Value + sample.Value = prometheusFloat64Ptr(value) + if prevMetric != nil && prevMetric.Counter != nil && prevMetric.Counter.Value != nil { + sample.Delta = prometheusFloat64Ptr(r.deltaFloat64(value, *prevMetric.Counter.Value)) + } else { + sample.Delta = prometheusFloat64Ptr(value) + } + } + return sample + } + + if metric.Histogram != nil { + sample.Type = "histogram" + histogram := metric.Histogram + if histogram.SampleSum != nil && !math.IsNaN(*histogram.SampleSum) { + sum := *histogram.SampleSum + sample.Sum = prometheusFloat64Ptr(sum) + if prevMetric != nil && prevMetric.Histogram != nil && prevMetric.Histogram.SampleSum != nil { + sample.SumDelta = prometheusFloat64Ptr(r.deltaFloat64(sum, *prevMetric.Histogram.SampleSum)) + } else { + sample.SumDelta = prometheusFloat64Ptr(sum) + } + } + if histogram.SampleCount != nil { + count := *histogram.SampleCount + sample.Count = prometheusUint64Ptr(count) + if prevMetric != nil && prevMetric.Histogram != nil && prevMetric.Histogram.SampleCount != nil { + countDelta := r.deltaUint64(count, *prevMetric.Histogram.SampleCount) + sample.CountDelta = &countDelta + } else { + sample.CountDelta = prometheusUint64Ptr(count) + } + } + return sample + } + + if metric.Summary != nil { + sample.Type = "summary" + summary := metric.Summary + if summary.SampleSum != nil && !math.IsNaN(*summary.SampleSum) { + sum := *summary.SampleSum + sample.Sum = prometheusFloat64Ptr(sum) + if prevMetric != nil && prevMetric.Summary != nil && prevMetric.Summary.SampleSum != nil { + sample.SumDelta = prometheusFloat64Ptr(r.deltaFloat64(sum, *prevMetric.Summary.SampleSum)) + } else { + sample.SumDelta = prometheusFloat64Ptr(sum) + } + } + if summary.SampleCount != nil { + count := *summary.SampleCount + sample.Count = prometheusUint64Ptr(count) + if prevMetric != nil && prevMetric.Summary != nil && prevMetric.Summary.SampleCount != nil { + countDelta := r.deltaUint64(count, *prevMetric.Summary.SampleCount) + sample.CountDelta = &countDelta + } else { + sample.CountDelta = prometheusUint64Ptr(count) + } + } + if len(summary.Quantile) > 0 { + sample.Quantiles = make(map[string]float64, len(summary.Quantile)) + for _, quantile := range summary.Quantile { + sample.Quantiles[r.formatQuantile(quantile.GetQuantile())] = quantile.GetValue() + } + } + return sample + } + + sample.Type = "unknown" + return sample +} + func (r *metricsCollector) prometheusMetricName(name string, metric *io_prometheus_client.Metric) string { labels := metric.GetLabel() if len(labels) == 0 { @@ -160,6 +307,19 @@ func (r *metricsCollector) prometheusMetricName(name string, metric *io_promethe return name + "_" + strings.Join(parts, "_") } +func (r *metricsCollector) prometheusLabels(metric *io_prometheus_client.Metric) map[string]string { + labels := metric.GetLabel() + if len(labels) == 0 { + return nil + } + + result := make(map[string]string, len(labels)) + for _, label := range labels { + result[label.GetName()] = label.GetValue() + } + return result +} + func (r *metricsCollector) addSummaryQuantiles(m *metrics.BlockMetrics, name string, metric *io_prometheus_client.Metric) { if metric.Summary == nil { return @@ -238,10 +398,25 @@ func (r *metricsCollector) deltaUint64(current uint64, previous uint64) uint64 { return current - previous } +func (r *metricsCollector) deltaFloat64(current float64, previous float64) float64 { + if current < previous { + return current + } + return current - previous +} + func (r *metricsCollector) formatQuantile(quantile float64) string { return strings.ReplaceAll(strconv.FormatFloat(quantile, 'f', -1, 64), ".", "_") } +func prometheusFloat64Ptr(value float64) *float64 { + return &value +} + +func prometheusUint64Ptr(value uint64) *uint64 { + return &value +} + func (r *metricsCollector) sanitizeMetricPart(value string) string { return strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { diff --git a/runner/clients/builder/metrics_test.go b/runner/clients/builder/metrics_test.go index af78a911..eb0310d4 100644 --- a/runner/clients/builder/metrics_test.go +++ b/runner/clients/builder/metrics_test.go @@ -72,10 +72,80 @@ func TestHistogramQuantileAvoidsInfiniteUpperBound(t *testing.T) { } } +func TestPrometheusMetricSamplePreservesLabelsAndHistogramDeltas(t *testing.T) { + collector := &metricsCollector{} + prev := histogramMetric( + 10, + bucket(1, 5), + bucket(2, 10), + ) + *prev.Histogram.SampleSum = 100 + + current := histogramMetric( + 16, + bucket(1, 8), + bucket(2, 16), + ) + *current.Histogram.SampleSum = 172 + current.Label = []*io_prometheus_client.LabelPair{ + {Name: stringPtr("flashblock_index"), Value: stringPtr("7")}, + } + + sample := collector.prometheusMetricSample( + "reth_base_builder_flashblock_txs_considered", + "reth_base_builder_flashblock_txs_considered_flashblock_index_7", + current, + prev, + ) + + if sample.Name != "reth_base_builder_flashblock_txs_considered" { + t.Fatalf("Name = %q", sample.Name) + } + if sample.Key != "reth_base_builder_flashblock_txs_considered_flashblock_index_7" { + t.Fatalf("Key = %q", sample.Key) + } + if sample.Labels["flashblock_index"] != "7" { + t.Fatalf("flashblock_index label = %q", sample.Labels["flashblock_index"]) + } + if sample.Count == nil || *sample.Count != 16 { + t.Fatalf("Count = %v, want 16", sample.Count) + } + if sample.CountDelta == nil || *sample.CountDelta != 6 { + t.Fatalf("CountDelta = %v, want 6", sample.CountDelta) + } + if sample.Sum == nil || *sample.Sum != 172 { + t.Fatalf("Sum = %v, want 172", sample.Sum) + } + if sample.SumDelta == nil || *sample.SumDelta != 72 { + t.Fatalf("SumDelta = %v, want 72", sample.SumDelta) + } +} + +func TestBuilderMetricTypesIncludeFlashblockDiagnostics(t *testing.T) { + collector := &metricsCollector{} + metricTypes := collector.GetMetricTypes() + + for _, name := range []string{ + "reth_base_builder_flashblock_txs_considered", + "reth_base_builder_flashblock_txs_included", + "reth_base_builder_flashblock_txs_rejected", + "reth_base_builder_flashblock_selection_total", + "reth_base_builder_flashblock_rejections_total", + "reth_base_builder_transaction_pool_fetch_duration", + "reth_base_builder_state_transition_merge_duration", + "reth_base_builder_flashblock_count", + } { + if !metricTypes[name] { + t.Fatalf("GetMetricTypes()[%q] = false, want true", name) + } + } +} + func histogramMetric(count uint64, buckets ...*io_prometheus_client.Bucket) *io_prometheus_client.Metric { return &io_prometheus_client.Metric{ Histogram: &io_prometheus_client.Histogram{ SampleCount: uint64Ptr(count), + SampleSum: float64Ptr(0), Bucket: buckets, }, } diff --git a/runner/metrics/metrics_interface.go b/runner/metrics/metrics_interface.go index 350dd3f9..59f56f59 100644 --- a/runner/metrics/metrics_interface.go +++ b/runner/metrics/metrics_interface.go @@ -19,18 +19,36 @@ type Collector interface { } type BlockMetrics struct { - BlockNumber uint64 - Timestamp time.Time - prevMetrics map[string]*io_prometheus_client.Metric - ExecutionMetrics map[string]interface{} + BlockNumber uint64 + Timestamp time.Time + prevMetrics map[string]*io_prometheus_client.Metric + ExecutionMetrics map[string]interface{} + PrometheusMetrics []PrometheusMetricSample `json:",omitempty"` +} + +// PrometheusMetricSample preserves a raw Prometheus sample alongside the +// flattened ExecutionMetrics values used by the report UI. +type PrometheusMetricSample struct { + Name string `json:"name"` + Key string `json:"key,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Type string `json:"type"` + Value *float64 `json:"value,omitempty"` + Delta *float64 `json:"delta,omitempty"` + Sum *float64 `json:"sum,omitempty"` + Count *uint64 `json:"count,omitempty"` + SumDelta *float64 `json:"sumDelta,omitempty"` + CountDelta *uint64 `json:"countDelta,omitempty"` + Quantiles map[string]float64 `json:"quantiles,omitempty"` } func NewBlockMetrics() *BlockMetrics { return &BlockMetrics{ - BlockNumber: 0, - prevMetrics: make(map[string]*io_prometheus_client.Metric), - ExecutionMetrics: make(map[string]interface{}), - Timestamp: time.Now(), + BlockNumber: 0, + prevMetrics: make(map[string]*io_prometheus_client.Metric), + ExecutionMetrics: make(map[string]interface{}), + PrometheusMetrics: make([]PrometheusMetricSample, 0), + Timestamp: time.Now(), } } @@ -43,14 +61,32 @@ func (m *BlockMetrics) Copy() *BlockMetrics { newPrevMetrics := make(map[string]*io_prometheus_client.Metric) maps.Copy(newMetrics, m.ExecutionMetrics) maps.Copy(newPrevMetrics, m.prevMetrics) + newPrometheusMetrics := make([]PrometheusMetricSample, len(m.PrometheusMetrics)) + for i, sample := range m.PrometheusMetrics { + newPrometheusMetrics[i] = sample.Copy() + } return &BlockMetrics{ - BlockNumber: m.BlockNumber, - prevMetrics: newPrevMetrics, - ExecutionMetrics: newMetrics, - Timestamp: m.Timestamp, + BlockNumber: m.BlockNumber, + prevMetrics: newPrevMetrics, + ExecutionMetrics: newMetrics, + PrometheusMetrics: newPrometheusMetrics, + Timestamp: m.Timestamp, } } +func (s PrometheusMetricSample) Copy() PrometheusMetricSample { + copied := s + if s.Labels != nil { + copied.Labels = make(map[string]string, len(s.Labels)) + maps.Copy(copied.Labels, s.Labels) + } + if s.Quantiles != nil { + copied.Quantiles = make(map[string]float64, len(s.Quantiles)) + maps.Copy(copied.Quantiles, s.Quantiles) + } + return copied +} + func (m *BlockMetrics) SetPreviousPrometheusMetrics(prev map[string]*io_prometheus_client.Metric) { m.prevMetrics = make(map[string]*io_prometheus_client.Metric, len(prev)) maps.Copy(m.prevMetrics, prev) @@ -158,6 +194,10 @@ func (m *BlockMetrics) AddExecutionMetric(name string, value interface{}) { m.ExecutionMetrics[name] = value } +func (m *BlockMetrics) AddPrometheusMetricSample(sample PrometheusMetricSample) { + m.PrometheusMetrics = append(m.PrometheusMetrics, sample) +} + func (m *BlockMetrics) GetMetricTypes() map[string]bool { return map[string]bool{ "execution": true, From b65b05bdc079b374cf4f94c3a2baee4abc7ea3e2 Mon Sep 17 00:00:00 2001 From: Julian Meyer Date: Fri, 29 May 2026 10:58:01 -0700 Subject: [PATCH 15/15] derive target_gps from gas limit and block time Co-authored-by: Sisyphus --- runner/benchmark/benchmark.go | 6 -- runner/benchmark/matrix_test.go | 70 ------------------- runner/network/types/types.go | 7 -- runner/payload/loadtest/load_test_worker.go | 14 ++-- .../payload/loadtest/load_test_worker_test.go | 6 +- 5 files changed, 13 insertions(+), 90 deletions(-) diff --git a/runner/benchmark/benchmark.go b/runner/benchmark/benchmark.go index 2e7631fd..48de407b 100644 --- a/runner/benchmark/benchmark.go +++ b/runner/benchmark/benchmark.go @@ -70,12 +70,6 @@ func NewParamsFromValues(assignments map[string]interface{}) (*types.RunParams, } else { return nil, fmt.Errorf("invalid gas limit %s", v) } - case "target_gps": - if vInt, ok := v.(int); ok { - params.TargetGPS = uint64(vInt) - } else { - return nil, fmt.Errorf("invalid target gps %s", v) - } case "consensus_timing": if vStr, ok := v.(string); ok { if vStr != "" && vStr != types.ConsensusTimingModePreventLateFCU && vStr != types.ConsensusTimingModeBaseConsensus { diff --git a/runner/benchmark/matrix_test.go b/runner/benchmark/matrix_test.go index adb99c4e..35d592e3 100644 --- a/runner/benchmark/matrix_test.go +++ b/runner/benchmark/matrix_test.go @@ -94,34 +94,6 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { }, wantErr: false, }, - { - name: "config with target gps", - config: benchmark.TestDefinition{ - Variables: []benchmark.Param{ - { - ParamType: "payload", - Value: "load-test", - }, - { - ParamType: "target_gps", - Value: 200_000_000, - }, - }, - }, - want: []benchmark.TestRun{ - { - Params: types.RunParams{ - NodeType: "geth", - PayloadID: "load-test", - GasLimit: benchmark.DefaultParams.GasLimit, - TargetGPS: 200_000_000, - BlockTime: 1 * time.Second, - ConsensusTimingMode: types.ConsensusTimingModePreventLateFCU, - }, - }, - }, - wantErr: false, - }, { name: "duplicate param type", config: benchmark.TestDefinition{ @@ -335,48 +307,6 @@ func TestNewTestPlanFromConfigAllowsSequencerThresholdsWithoutValidator(t *testi require.False(t, plan.Mode.RunValidator) } -func TestResolveTestRunsFromMatrixExpandsTargetGPSValues(t *testing.T) { - config := &benchmark.BenchmarkConfig{Name: "snapshot load test"} - definition := benchmark.TestDefinition{ - Variables: []benchmark.Param{ - { - ParamType: "payload", - Value: "mainnet-snapshot-load-test", - }, - { - ParamType: "node_type", - Value: "builder", - }, - { - ParamType: "gas_limit", - Value: 1_200_000_000, - }, - { - ParamType: "target_gps", - Values: []interface{}{ - 80_000_000, - 400_000_000, - 1_200_000_000, - }, - }, - }, - } - - runs, err := benchmark.ResolveTestRunsFromMatrix(definition, "snapshot-load-test.yml", config) - require.NoError(t, err) - require.Len(t, runs, 3) - - require.Equal(t, uint64(1_200_000_000), runs[0].Params.GasLimit) - require.Equal(t, uint64(1_200_000_000), runs[1].Params.GasLimit) - require.Equal(t, uint64(1_200_000_000), runs[2].Params.GasLimit) - require.Equal(t, uint64(80_000_000), runs[0].Params.TargetGPS) - require.Equal(t, uint64(400_000_000), runs[1].Params.TargetGPS) - require.Equal(t, uint64(1_200_000_000), runs[2].Params.TargetGPS) - require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[0].Params.ConsensusTimingMode) - require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[1].Params.ConsensusTimingMode) - require.Equal(t, types.ConsensusTimingModePreventLateFCU, runs[2].Params.ConsensusTimingMode) -} - func TestResolveTestRunsFromMatrixDefaultsSnapshotLoadTestsToBaseConsensusTiming(t *testing.T) { config := &benchmark.BenchmarkConfig{ Name: "snapshot load test", diff --git a/runner/network/types/types.go b/runner/network/types/types.go index 8f78d9e7..ee684344 100644 --- a/runner/network/types/types.go +++ b/runner/network/types/types.go @@ -66,9 +66,6 @@ type RunParams struct { // GasLimit is the gas limit for the benchmark run which is the maximum gas that the sequencer will include per block. GasLimit uint64 - // TargetGPS is the target gas per second for load-test payloads. - TargetGPS uint64 - // PayloadID is a reference to a transaction payload that will be sent to the sequencer. PayloadID string @@ -127,10 +124,6 @@ func (p RunParams) ToConfig() map[string]interface{} { params["ValidatorNodeType"] = p.ValidatorNodeType } - if p.TargetGPS > 0 { - params["TargetGPS"] = p.TargetGPS - } - if p.ConsensusTimingMode != "" { params["ConsensusTimingMode"] = p.ConsensusTimingMode } diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go index b59bb333..2158ddde 100644 --- a/runner/payload/loadtest/load_test_worker.go +++ b/runner/payload/loadtest/load_test_worker.go @@ -37,7 +37,8 @@ type loadTestPayloadWorker struct { loadTestBin string elRPCURL string flashblocksURL string - targetGPS uint64 + gasLimit uint64 + blockTime time.Duration params LoadTestPayloadDefinition mempool *mempool.StaticWorkloadMempool cmd *exec.Cmd @@ -78,7 +79,8 @@ func NewLoadTestPayloadWorker( loadTestBin: cfg.LoadTestBinary(), elRPCURL: elRPCURL, flashblocksURL: flashblocksURL, - targetGPS: params.TargetGPS, + gasLimit: params.GasLimit, + blockTime: params.BlockTime, params: definition, mempool: mp, done: make(chan struct{}), @@ -254,8 +256,9 @@ func (w *loadTestPayloadWorker) buildConfig() (*yaml.Node, error) { flashblocksURL = "ws://localhost:7111" } setMappingValue(config, "flashblocks_ws", stringNode(flashblocksURL)) - if w.targetGPS > 0 { - setMappingValue(config, "target_gps", uintNode(w.targetGPS)) + if w.blockTime > 0 && w.gasLimit > 0 { + targetGPS := w.gasLimit / uint64(w.blockTime.Seconds()) + setMappingValue(config, "target_gps", uintNode(targetGPS)) } return config, nil @@ -330,7 +333,8 @@ func (w *loadTestPayloadWorker) writeConfig() (string, error) { w.log.Info("Generated load-test config", "source_config", w.sourceConfigPath, - "target_gps", w.targetGPS, + "gas_limit", w.gasLimit, + "block_time", w.blockTime, ) return tmpFile.Name(), nil diff --git a/runner/payload/loadtest/load_test_worker_test.go b/runner/payload/loadtest/load_test_worker_test.go index ec59b8e3..a0f706d4 100644 --- a/runner/payload/loadtest/load_test_worker_test.go +++ b/runner/payload/loadtest/load_test_worker_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" @@ -67,7 +68,8 @@ transactions: worker := &loadTestPayloadWorker{ flashblocksURL: "ws://benchmark-flashblocks.example", - targetGPS: 75_000_000, + gasLimit: 150_000_000, + blockTime: 2 * time.Second, elRPCURL: "http://sequencer.example", sourceConfigPath: configPath, } @@ -109,7 +111,7 @@ transactions: } } -func TestBuildConfigPreservesNativeTargetGPSWhenBenchmarkTargetUnset(t *testing.T) { +func TestBuildConfigPreservesNativeTargetGPSWhenGasLimitOrBlockTimeIsZero(t *testing.T) { configPath := filepath.Join(t.TempDir(), "load-test.yaml") err := os.WriteFile(configPath, []byte(` transaction_submission_rpcs: