From 85fa5b3e0f1935ee94c72632497d2c0adcab9083 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 3 Jul 2026 15:15:21 -0300 Subject: [PATCH 1/3] perf(stark): fuse deep-composition reconstruction for both FRI points Reconstructing the deep composition polynomial per query walked the OOD table, the trace-term coefficients, and inverted the trace/comp denominators independently for the regular and symmetric evaluation points, even though the OOD-derived terms don't depend on the point. Rewriting coeff*(base-ood)*denom as denom*(coeff*base - coeff*ood) isolates the point-independent coeff*ood term (computed once, shared between both points) from coeff*base (cheap base*ext multiply for base-field columns), and lets both denom sets batch-invert together. Multi-query recursion profile: 2,210,366,539 -> 1,951,764,531 cycles (-11.7%); step 3 (FRI) 912M -> 652M cycles (-28.5%). --- crypto/stark/src/verifier.rs | 179 +++++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 7ab1843a2..676fd311c 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -636,18 +636,6 @@ where .map(|a| evals(&a.evaluations)) .unwrap_or(&[]); - let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, false, domain); - deep_poly_evaluations.push(Self::reconstruct_deep_composition_poly_evaluation( - proof, - &evaluation_point, - primitive_root, - challenges, - &lde_base, - lde_aux, - evals(&opening.composition_poly.evaluations), - )?); - - // Mirror for the symmetric query point. let mut lde_base_sym: Vec> = Vec::new(); if let Some(p) = opening.precomputed_trace_polys.as_ref() { lde_base_sym.extend_from_slice(evals(&p.evaluations_sym)); @@ -660,29 +648,60 @@ where .map(|a| evals(&a.evaluations_sym)) .unwrap_or(&[]); - let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, true, domain); - deep_poly_evaluations_sym.push(Self::reconstruct_deep_composition_poly_evaluation( - proof, - &evaluation_point, - primitive_root, - challenges, - &lde_base_sym, - lde_aux_sym, - evals(&opening.composition_poly.evaluations_sym), - )?); + let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, false, domain); + let evaluation_point_sym = + Self::query_challenge_to_evaluation_point(*iota, true, domain); + let (evaluation, evaluation_sym) = + Self::reconstruct_deep_composition_poly_evaluation_pair( + proof, + &evaluation_point, + &evaluation_point_sym, + primitive_root, + challenges, + &lde_base, + lde_aux, + evals(&opening.composition_poly.evaluations), + &lde_base_sym, + lde_aux_sym, + evals(&opening.composition_poly.evaluations_sym), + )?; + deep_poly_evaluations.push(evaluation); + deep_poly_evaluations_sym.push(evaluation_sym); } Some((deep_poly_evaluations, deep_poly_evaluations_sym)) } - fn reconstruct_deep_composition_poly_evaluation( + /// Reconstructs the deep composition polynomial evaluation at a query's + /// evaluation point and its symmetric counterpart in one traversal. The + /// OOD table walk and `trace_term_coeffs` walk are shared between the two + /// (they don't depend on the evaluation point), halving the loads and + /// letting the two `denoms_trace` vectors, and the two composition-tail + /// denominators, share a single batch-inverse call each. + /// + /// Algebraic rewrite of the per-element trace term: for column `col` and + /// row `row`, + /// coeff(col,row) * (base(col) - ood(row,col)) * denom(row) + /// distributes to + /// denom(row) * (coeff(col,row) * base(col) - coeff(col,row) * ood(row,col)). + /// Summing over `col` first isolates `coeff * ood`, which is identical for + /// both evaluation points, from `coeff * base`, which differs; and moves + /// the `denom(row)` multiplication (ext * ext) out of the column loop. + /// When `col` indexes a base-field column this also lets `coeff * base` + /// use the cheap `IsSubFieldOf` asymmetric multiplication instead of a + /// full extension-field product. + fn reconstruct_deep_composition_poly_evaluation_pair( proof: &ArchivedStarkProof, evaluation_point: &FieldElement, + evaluation_point_sym: &FieldElement, primitive_root: &FieldElement, challenges: &Challenges, lde_trace_base_evaluations: &[FieldElement], lde_trace_aux_evaluations: &[FieldElement], lde_composition_poly_parts_evaluation: &[FieldElement], - ) -> Option> { + lde_trace_base_evaluations_sym: &[FieldElement], + lde_trace_aux_evaluations_sym: &[FieldElement], + lde_composition_poly_parts_evaluation_sym: &[FieldElement], + ) -> Option<(FieldElement, FieldElement)> { let ood_evaluations_table_height = proof.trace_ood_evaluations.height(); let ood_evaluations_table_width = proof.trace_ood_evaluations.width(); // Hot loop below: resolve the archived OOD data to one flat slice once @@ -690,12 +709,18 @@ where let ood_data = proof.trace_ood_evaluations.row_major_data(); let trace_term_coeffs = &challenges.trace_term_coeffs; - // Runtime guard: a malformed proof may supply opening evaluations whose - // column count does not match the OOD table width, or whose composition - // poly parts count does not match the proof's `composition_poly_parts_ood_evaluation`. - // Without these checks the indexing below would panic in release builds. - if lde_trace_base_evaluations.len() + lde_trace_aux_evaluations.len() - != ood_evaluations_table_width + let num_base = lde_trace_base_evaluations.len(); + // Runtime guards: a malformed proof may supply opening evaluations + // whose column count does not match the OOD table width, whose + // regular/symmetric base-column split disagree, or whose composition + // poly parts count does not match the proof's + // `composition_poly_parts_ood_evaluation`. Without these checks the + // indexing below would panic in release builds. + if num_base != lde_trace_base_evaluations_sym.len() { + return None; + } + if num_base + lde_trace_aux_evaluations.len() != ood_evaluations_table_width + || num_base + lde_trace_aux_evaluations_sym.len() != ood_evaluations_table_width { return None; } @@ -706,54 +731,86 @@ where return None; } - let mut denoms_trace = Vec::with_capacity(ood_evaluations_table_height); + // Build both denominator sets (regular, then symmetric) and invert + // them together in a single batch. + let mut denoms = Vec::with_capacity(2 * ood_evaluations_table_height); let mut current_z = challenges.z.clone(); for _ in 0..ood_evaluations_table_height { - denoms_trace.push(evaluation_point - ¤t_z); + denoms.push(evaluation_point - ¤t_z); + current_z = primitive_root * ¤t_z; + } + let mut current_z = challenges.z.clone(); + for _ in 0..ood_evaluations_table_height { + denoms.push(evaluation_point_sym - ¤t_z); current_z = primitive_root * ¤t_z; } // A malformed proof can land an OOD evaluation point on the LDE coset, reject. - FieldElement::inplace_batch_inverse(&mut denoms_trace).ok()?; - - let num_base = lde_trace_base_evaluations.len(); - let trace_term = (0..ood_evaluations_table_width) - .zip(&challenges.trace_term_coeffs) - .fold(FieldElement::zero(), |trace_terms, (col_idx, coeff_row)| { - let trace_i = (0..ood_evaluations_table_height).zip(coeff_row).fold( - FieldElement::zero(), - |trace_t, (row_idx, coeff)| { - let ood_val = &ood_data[row_idx * ood_evaluations_table_width + col_idx]; - // Stay in base when we can: F: IsSubFieldOf gives F - E -> E. - let diff: FieldElement = if col_idx < num_base { - &lde_trace_base_evaluations[col_idx] - ood_val - } else { - &lde_trace_aux_evaluations[col_idx - num_base] - ood_val - }; - let poly_evaluation = diff * &denoms_trace[row_idx]; - trace_t + &poly_evaluation * coeff - }, - ); - trace_terms + trace_i - }); + FieldElement::inplace_batch_inverse(&mut denoms).ok()?; + let (denoms_trace, denoms_trace_sym) = denoms.split_at(ood_evaluations_table_height); + + let mut trace_term = FieldElement::::zero(); + let mut trace_term_sym = FieldElement::::zero(); + for row_idx in 0..ood_evaluations_table_height { + let ood_row = &ood_data[row_idx * ood_evaluations_table_width + ..(row_idx + 1) * ood_evaluations_table_width]; + let mut ood_row_sum = FieldElement::::zero(); + let mut base_row_sum = FieldElement::::zero(); + let mut base_row_sum_sym = FieldElement::::zero(); + for col_idx in 0..ood_evaluations_table_width { + let coeff = &trace_term_coeffs[col_idx][row_idx]; + let ood_val = &ood_row[col_idx]; + ood_row_sum = ood_row_sum + coeff * ood_val; + if col_idx < num_base { + // F: IsSubFieldOf gives the cheap asymmetric F * E -> E product. + base_row_sum = base_row_sum + &lde_trace_base_evaluations[col_idx] * coeff; + base_row_sum_sym = + base_row_sum_sym + &lde_trace_base_evaluations_sym[col_idx] * coeff; + } else { + let aux_idx = col_idx - num_base; + base_row_sum = base_row_sum + coeff * &lde_trace_aux_evaluations[aux_idx]; + base_row_sum_sym = + base_row_sum_sym + coeff * &lde_trace_aux_evaluations_sym[aux_idx]; + } + } + trace_term = trace_term + &denoms_trace[row_idx] * &(&base_row_sum - &ood_row_sum); + trace_term_sym = + trace_term_sym + &denoms_trace_sym[row_idx] * &(&base_row_sum_sym - &ood_row_sum); + } let composition_parts_ood = evals(&proof.composition_poly_parts_ood_evaluation); let number_of_parts = lde_composition_poly_parts_evaluation.len(); + // A malformed proof can give the regular and symmetric openings a + // different composition-poly-parts count; reject rather than let the + // shared `z_pow`/bounds checks below silently apply the wrong count + // to one side. + if number_of_parts != lde_composition_poly_parts_evaluation_sym.len() { + return None; + } let z_pow = &challenges.z.pow(number_of_parts); // A malformed proof can make evaluation_point == z^N, reject. - let denom_composition = (evaluation_point - z_pow).inv().ok()?; - let mut h_terms = FieldElement::zero(); - for (j, h_i_upsilon) in lde_composition_poly_parts_evaluation.iter().enumerate() { + let mut denom_composition_pair = [evaluation_point - z_pow, evaluation_point_sym - z_pow]; + FieldElement::inplace_batch_inverse(&mut denom_composition_pair).ok()?; + let [denom_composition, denom_composition_sym] = denom_composition_pair; + + let mut h_sum_zpow = FieldElement::::zero(); + let mut h_sum = FieldElement::::zero(); + let mut h_sum_sym = FieldElement::::zero(); + for j in 0..number_of_parts { + let h_i_upsilon = &lde_composition_poly_parts_evaluation[j]; + let h_i_upsilon_sym = &lde_composition_poly_parts_evaluation_sym[j]; // Bounds-check via `.get(j)?`: a malformed opening may have more // parts than the proof header advertises. let h_i_zpower = composition_parts_ood.get(j)?; let gamma = challenges.gammas.get(j)?; - let h_i_term = (h_i_upsilon - h_i_zpower) * gamma; - h_terms += h_i_term; + h_sum_zpow = h_sum_zpow + h_i_zpower * gamma; + h_sum = h_sum + h_i_upsilon * gamma; + h_sum_sym = h_sum_sym + h_i_upsilon_sym * gamma; } - h_terms *= denom_composition; + let h_terms = (&h_sum - &h_sum_zpow) * denom_composition; + let h_terms_sym = (&h_sum_sym - &h_sum_zpow) * denom_composition_sym; - Some(trace_term + h_terms) + Some((trace_term + h_terms, trace_term_sym + h_terms_sym)) } /// Verifies one or more STARK proofs with their corresponding AIRs. From 07db8f0de70539ce0443e5eb507fe78dbc0f8d0b Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 3 Jul 2026 15:40:13 -0300 Subject: [PATCH 2/3] docs(stark): drop duplicate IsSubFieldOf note from doc comment The doc comment on reconstruct_deep_composition_poly_evaluation_pair restated the inline comment three lines into the function body. --- crypto/stark/src/verifier.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 676fd311c..1bfaabb16 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -686,9 +686,6 @@ where /// Summing over `col` first isolates `coeff * ood`, which is identical for /// both evaluation points, from `coeff * base`, which differs; and moves /// the `denom(row)` multiplication (ext * ext) out of the column loop. - /// When `col` indexes a base-field column this also lets `coeff * base` - /// use the cheap `IsSubFieldOf` asymmetric multiplication instead of a - /// full extension-field product. fn reconstruct_deep_composition_poly_evaluation_pair( proof: &ArchivedStarkProof, evaluation_point: &FieldElement, From 3ac5e494d24996659aca8553d49e2869f7758684 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 3 Jul 2026 16:35:10 -0300 Subject: [PATCH 3/3] perf(stark): dedupe z-ladder walk, alloc-free composition-tail inverse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reconstruct_deep_composition_poly_evaluation_pair walked the z*g^k ladder twice per query (once per evaluation point) though it doesn't depend on which point is being evaluated; interleave the two denominator pushes into one walk instead. The 2-element composition- tail batch inverse also went through the general Vec-allocating inplace_batch_inverse; hand-roll it (one inversion, three muls, no allocation). Also mark the IsSubFieldOf mul/ add/sub impls for GoldilocksField #[inline(always)], matching the sibling IsField impls on the same types — these are exactly the cheap asymmetric ops the deep-composition reconstruction fusion relies on. Multi-query recursion profile (two make-rebuilt runs per side, since the underlying proof has run-to-run variance): baseline 1,947,221,680 / 1,947,263,739 cycles; with this change 1,939,869,854 / 1,939,845,966 (-~7.37M, -0.38%). No correctness change (in-VM verify output digest identical to the prior commit). --- .../math/src/field/extensions_goldilocks.rs | 6 ++++ crypto/stark/src/verifier.rs | 29 ++++++++++--------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crypto/math/src/field/extensions_goldilocks.rs b/crypto/math/src/field/extensions_goldilocks.rs index 45fd7274b..feb5c69ed 100644 --- a/crypto/math/src/field/extensions_goldilocks.rs +++ b/crypto/math/src/field/extensions_goldilocks.rs @@ -199,6 +199,7 @@ impl IsField for Degree2GoldilocksExtensionField { } impl IsSubFieldOf for GoldilocksField { + #[inline(always)] fn mul( a: &Self::BaseType, b: &::BaseType, @@ -208,6 +209,7 @@ impl IsSubFieldOf for GoldilocksField { [c0, c1] } + #[inline(always)] fn add( a: &Self::BaseType, b: &::BaseType, @@ -224,6 +226,7 @@ impl IsSubFieldOf for GoldilocksField { Ok(>::mul(a, &b_inv)) } + #[inline(always)] fn sub( a: &Self::BaseType, b: &::BaseType, @@ -410,6 +413,7 @@ impl IsField for Degree3GoldilocksExtensionField { } impl IsSubFieldOf for GoldilocksField { + #[inline(always)] fn mul( a: &Self::BaseType, b: &::BaseType, @@ -420,6 +424,7 @@ impl IsSubFieldOf for GoldilocksField { [c0, c1, c2] } + #[inline(always)] fn add( a: &Self::BaseType, b: &::BaseType, @@ -436,6 +441,7 @@ impl IsSubFieldOf for GoldilocksField { Ok(>::mul(a, &b_inv)) } + #[inline(always)] fn sub( a: &Self::BaseType, b: &::BaseType, diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 1bfaabb16..0cf5eb460 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -728,22 +728,18 @@ where return None; } - // Build both denominator sets (regular, then symmetric) and invert - // them together in a single batch. + // Build both denominator sets in one walk of the shared z*g^k ladder, + // interleaved (regular, symmetric, regular, symmetric, ...), and + // invert them together in a single batch. let mut denoms = Vec::with_capacity(2 * ood_evaluations_table_height); let mut current_z = challenges.z.clone(); for _ in 0..ood_evaluations_table_height { denoms.push(evaluation_point - ¤t_z); - current_z = primitive_root * ¤t_z; - } - let mut current_z = challenges.z.clone(); - for _ in 0..ood_evaluations_table_height { denoms.push(evaluation_point_sym - ¤t_z); current_z = primitive_root * ¤t_z; } // A malformed proof can land an OOD evaluation point on the LDE coset, reject. FieldElement::inplace_batch_inverse(&mut denoms).ok()?; - let (denoms_trace, denoms_trace_sym) = denoms.split_at(ood_evaluations_table_height); let mut trace_term = FieldElement::::zero(); let mut trace_term_sym = FieldElement::::zero(); @@ -769,9 +765,9 @@ where base_row_sum_sym + coeff * &lde_trace_aux_evaluations_sym[aux_idx]; } } - trace_term = trace_term + &denoms_trace[row_idx] * &(&base_row_sum - &ood_row_sum); - trace_term_sym = - trace_term_sym + &denoms_trace_sym[row_idx] * &(&base_row_sum_sym - &ood_row_sum); + trace_term = trace_term + &denoms[2 * row_idx] * &(&base_row_sum - &ood_row_sum); + trace_term_sym = trace_term_sym + + &denoms[2 * row_idx + 1] * &(&base_row_sum_sym - &ood_row_sum); } let composition_parts_ood = evals(&proof.composition_poly_parts_ood_evaluation); @@ -785,10 +781,15 @@ where } let z_pow = &challenges.z.pow(number_of_parts); - // A malformed proof can make evaluation_point == z^N, reject. - let mut denom_composition_pair = [evaluation_point - z_pow, evaluation_point_sym - z_pow]; - FieldElement::inplace_batch_inverse(&mut denom_composition_pair).ok()?; - let [denom_composition, denom_composition_sym] = denom_composition_pair; + // A malformed proof can make evaluation_point == z^N, reject. Hand-rolled + // 2-element batch inverse (one inversion, three muls, no allocation): + // for a, b with product p = a*b, inv(p) * b = inv(a) and inv(p) * a = inv(b). + let denom_composition_raw = evaluation_point - z_pow; + let denom_composition_sym_raw = evaluation_point_sym - z_pow; + let denom_composition_prod_inv = + (&denom_composition_raw * &denom_composition_sym_raw).inv().ok()?; + let denom_composition = &denom_composition_prod_inv * &denom_composition_sym_raw; + let denom_composition_sym = &denom_composition_prod_inv * &denom_composition_raw; let mut h_sum_zpow = FieldElement::::zero(); let mut h_sum = FieldElement::::zero();