Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,10 @@ function normalizeCompositeWeights(weights: FindingSeverityCalibrationWeights |
),
};
const total = raw.objectiveAnchor + raw.pairwiseJudge + raw.structuredFindingSeverity;
if (total <= 0) return DEFAULT_COMPOSITE_WEIGHTS;
// Preserve explicitly-zeroed weights rather than substituting the defaults: a caller that zeroes every component
// must reach the objective-only fallback in the composite scorer, not silently get the default 45/35/20 blend
// (converges with reviewer-consensus-calibration.ts's already-correct behavior; #6170).
if (total <= 0) return { objectiveAnchor: 0, pairwiseJudge: 0, structuredFindingSeverity: 0 };
return {
objectiveAnchor: raw.objectiveAnchor / total,
pairwiseJudge: raw.pairwiseJudge / total,
Expand Down
8 changes: 6 additions & 2 deletions packages/loopover-engine/src/gate-verdict-calibration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ function sanitizeGateVerdictCalibrationIngestion(

for (const row of ingestion.rejected) {
if (!isRecord(row)) continue;
const repoFullName = typeof row.repoFullName === "string" ? normalizeRepoFullName(row.repoFullName) : null;
const repoFullName =
typeof row.repoFullName === "string" ? (normalizeRepoFullName(row.repoFullName) ?? normalizeId(row.repoFullName)) : null;
const replayRunId = typeof row.replayRunId === "string" ? normalizeId(row.replayRunId) : null;
const gateRunId = typeof row.gateRunId === "string" ? normalizeId(row.gateRunId) : null;
const reason = row.reason;
Expand Down Expand Up @@ -315,7 +316,10 @@ function normalizeCompositeWeights(weights: GateVerdictCalibrationWeights | unde
),
};
const total = raw.objectiveAnchor + raw.pairwiseJudge + raw.structuredGateVerdict;
if (total <= 0) return DEFAULT_COMPOSITE_WEIGHTS;
// Preserve explicitly-zeroed weights rather than substituting the defaults: a caller that zeroes every component
// must reach the objective-only fallback in the composite scorer, not silently get the default 45/35/20 blend
// (converges with reviewer-consensus-calibration.ts's already-correct behavior; #6170).
if (total <= 0) return { objectiveAnchor: 0, pairwiseJudge: 0, structuredGateVerdict: 0 };
return {
objectiveAnchor: raw.objectiveAnchor / total,
pairwiseJudge: raw.pairwiseJudge / total,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -470,3 +470,16 @@ test("renderAuditMarkdown tolerates malformed pre-ingested rejected rows after s
assert.deepEqual(result.audit.rejected, []);
assert.doesNotThrow(() => renderFindingSeverityCalibrationAuditMarkdown(result));
});

test("computeFindingSeverityCompositeCalibrationScore falls back to objective-only when all weights are explicitly zero (#6170)", () => {
const result = computeFindingSeverityCompositeCalibrationScore({
objectiveAnchor: 0.4,
pairwise: 0.4,
findingSeverity: [signal()],
weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredFindingSeverity: 0 },
});
// Explicitly zeroing every component falls back to objective-only -- NOT the default 45/35/20 blend
// (converges with reviewer-consensus-calibration.ts).
assert.deepEqual(result.weights, { objectiveAnchor: 1, pairwiseJudge: 0, structuredFindingSeverity: 0 });
assert.equal(result.compositeScore, 0.4);
});
41 changes: 41 additions & 0 deletions packages/loopover-engine/test/gate-verdict-calibration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -580,3 +580,44 @@ test("computeGateVerdictCompositeCalibrationScore ignores malformed pre-ingested
assert.deepEqual(result.audit.contributingRepos, []);
assert.deepEqual(result.audit.rejected, []);
});

test("computeGateVerdictCompositeCalibrationScore falls back to objective-only when all weights are explicitly zero (#6170)", () => {
const result = computeGateVerdictCompositeCalibrationScore({
objectiveAnchor: 0.4,
pairwise: 0.4,
gateVerdicts: [
{
repoFullName: "acme/widgets",
replayRunId: "replay-1",
gateRunId: "gate-1",
optedIn: true,
dimensions: [{ dimension: "correctness", outcome: "pass" }],
},
],
weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredGateVerdict: 0 },
});
// Explicitly zeroing every component falls back to objective-only -- NOT the default 45/35/20 blend
// (converges with reviewer-consensus-calibration.ts).
assert.deepEqual(result.weights, { objectiveAnchor: 1, pairwiseJudge: 0, structuredGateVerdict: 0 });
assert.equal(result.compositeScore, 0.4);
});

test("computeGateVerdictCompositeCalibrationScore preserves a malformed-repo (invalid_repo) rejected row instead of dropping it (#6170)", () => {
const result = computeGateVerdictCompositeCalibrationScore({
objectiveAnchor: 0.5,
pairwise: 0.5,
gateVerdicts: {
accepted: [],
rejected: [
{ repoFullName: "acme/widgets", replayRunId: "replay-1", gateRunId: "gate-1", reason: "not_opted_in" },
{ repoFullName: "bad", replayRunId: "replay-2", gateRunId: "gate-2", reason: "invalid_repo" },
],
} as never,
});
// "bad" is not a valid owner/repo (normalizeRepoFullName -> null); the `?? normalizeId` fallback preserves the
// raw string instead of dropping the row, matching finding-severity + reviewer-consensus.
assert.deepEqual(result.audit.rejected, [
{ repoFullName: "acme/widgets", replayRunId: "replay-1", gateRunId: "gate-1", reason: "not_opted_in" },
{ repoFullName: "bad", replayRunId: "replay-2", gateRunId: "gate-2", reason: "invalid_repo" },
]);
});
64 changes: 64 additions & 0 deletions test/unit/engine-calibration-convergence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import {
computeGateVerdictCompositeCalibrationScore,
computeFindingSeverityCompositeCalibrationScore,
} from "../../packages/loopover-engine/src/index";

// Converges gate-verdict + finding-severity calibration with reviewer-consensus-calibration.ts's already-correct
// all-zero-weight + malformed-repo handling (#6170). These run under vitest (Codecov-measured) so the changed
// engine-src branches are covered; the engine's own node:test suites carry the equivalent assertions.
describe("gate-verdict/finding-severity calibration convergence (#6170)", () => {
it("gate-verdict: all-zero weights fall back to objective-only, not the default 45/35/20 blend", () => {
const result = computeGateVerdictCompositeCalibrationScore({
objectiveAnchor: 0.4,
pairwise: 0.4,
gateVerdicts: [
{
repoFullName: "acme/widgets",
replayRunId: "replay-1",
gateRunId: "gate-1",
optedIn: true,
dimensions: [{ dimension: "correctness", outcome: "pass" }],
},
],
weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredGateVerdict: 0 },
});
expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0, structuredGateVerdict: 0 });
expect(result.compositeScore).toBe(0.4);
});

it("finding-severity: all-zero weights fall back to objective-only, not the default blend", () => {
const result = computeFindingSeverityCompositeCalibrationScore({
objectiveAnchor: 0.4,
pairwise: 0.4,
findingSeverity: [
{ repoFullName: "acme/widgets", replayRunId: "replay-1", reviewRunId: "review-1", optedIn: true, tiers: [{ tier: "blocker", total: 2, confirmed: 2 }] },
],
weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredFindingSeverity: 0 },
});
expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0, structuredFindingSeverity: 0 });
expect(result.compositeScore).toBe(0.4);
});

it("gate-verdict: preserves a malformed-repo (invalid_repo) rejected row instead of dropping it; keeps a valid repo and drops a non-string one", () => {
const result = computeGateVerdictCompositeCalibrationScore({
objectiveAnchor: 0.5,
pairwise: 0.5,
gateVerdicts: {
accepted: [],
rejected: [
{ repoFullName: "acme/widgets", replayRunId: "replay-1", gateRunId: "gate-1", reason: "not_opted_in" },
{ repoFullName: "bad", replayRunId: "replay-2", gateRunId: "gate-2", reason: "invalid_repo" },
{ repoFullName: 123, replayRunId: "replay-3", gateRunId: "gate-3", reason: "invalid_repo" },
],
} as never,
});
// "bad" is not a valid owner/repo, so normalizeRepoFullName returns null; the `?? normalizeId` fallback
// preserves the raw string instead of dropping the row (matching reviewer-consensus). The non-string repo
// (123) still drops (ternary false branch).
expect(result.audit.rejected).toEqual([
{ repoFullName: "acme/widgets", replayRunId: "replay-1", gateRunId: "gate-1", reason: "not_opted_in" },
{ repoFullName: "bad", replayRunId: "replay-2", gateRunId: "gate-2", reason: "invalid_repo" },
]);
});
});