diff --git a/changelog.d/fixes/release-green-complexity-outcomes.md b/changelog.d/fixes/release-green-complexity-outcomes.md new file mode 100644 index 0000000000..d67c5c1b5e --- /dev/null +++ b/changelog.d/fixes/release-green-complexity-outcomes.md @@ -0,0 +1 @@ +- **fix(quality):** prevent cancelled, failed, or incomplete complexity measurements from appearing as passing release-green metrics; retain advisory classification for complete, measured ratchet drift without changing baselines or CI workflows. diff --git a/scripts/quality/validate-release-green.mjs b/scripts/quality/validate-release-green.mjs index 2c5bdb0022..7dc1bc4def 100644 --- a/scripts/quality/validate-release-green.mjs +++ b/scripts/quality/validate-release-green.mjs @@ -243,6 +243,59 @@ export function parseCognitiveCount(out) { return m ? Number(m[1]) : null; } +/** Translate the shared complexity walk into the two metric records. */ +export function evaluateComplexityRun({ code, out }, cogBase) { + // The shared walk emits both machine counts followed by both ratchet verdicts. + // Do not use the legacy generic "N violações" fallback here: a partial walk + // containing only the cyclomatic summary would masquerade as cognitive data. + const cogMatch = /^cognitiveComplexity=(\d+)\r?$/m.exec(out); + const cycMatch = /^complexity=(\d+)\r?$/m.exec(out); + const cogSummary = /^\[cognitive-complexity\] (OK|REGRESSÃO) — (\d+) violações /m.exec(out); + const cycSummary = /^\[complexity\] (OK|REGRESSÃO) — (\d+) violações /m.exec(out); + const cogCurrent = cogMatch ? Number(cogMatch[1]) : null; + const cycCurrent = cycMatch ? Number(cycMatch[1]) : null; + const cogOver = isDrift(cogCurrent, cogBase); + const cycRegressed = cycSummary?.[1] === "REGRESSÃO"; + const measured = + [cogCurrent, cycCurrent, cogBase].every((count) => Number.isSafeInteger(count) && count >= 0) && + Number(cogSummary?.[2]) === cogCurrent && + Number(cycSummary?.[2]) === cycCurrent && + (cogSummary?.[1] === "REGRESSÃO") === cogOver; + // Exit 1 is expected only for a complete, measured ratchet regression. A + // cancelled/crashed tool or missing metrics is execution failure, not drift. + const measuredDrift = code === 1 && measured && (cogOver || cycRegressed); + if (!measured || (code !== 0 && !measuredDrift)) { + return [ + { + id: "complexity", + label: "Complexity measurement", + kind: "hard", + ok: false, + detail: + code !== 0 + ? `Complexity process exited ${code}: ${firstFailureLine(out)}` + : "Complexity process did not report complete finite metrics and baseline", + }, + ]; + } + return [ + { + id: "cognitive-complexity", + label: "Cognitive complexity (ratchet)", + kind: "drift", + ok: !cogOver, + detail: `${cogCurrent} vs baseline ${cogBase}${cogOver ? ` (+${cogCurrent - cogBase} drift → rebaseline at release)` : ""}`, + }, + { + id: "complexity", + label: "Cyclomatic complexity (ratchet)", + kind: "drift", + ok: !cycRegressed, + detail: `complexity=${cycCurrent} (shared walk with cognitive)${cycRegressed ? " REGRESSED" : ""}`, + }, + ]; +} + /** * Drift verdict for a ratchet: a metric that grew past its committed baseline is * "drift" (reported, never blocking). `direction:"down"` metrics (warnings, @@ -591,42 +644,11 @@ async function main() { // Complexity + cognitive (one ESLint walk; both still recorded as drift) { announce("Complexity + cognitive ratchets (shared ESLint walk)"); - const { out } = run(npmCmd, ["run", "check:complexity-ratchets"]); - saveGateLog("complexity-ratchets", out); - const cogCurrent = parseCognitiveCount(out); - const cogBase = baselineValue("cognitiveComplexity"); - const cogOver = isDrift(cogCurrent, cogBase); - const cycMatch = /(?:^|\n)complexity=(\d+)/.exec(out); - const cycOkMatch = /\[complexity\] OK — (\d+)/.exec(out); - const cycRegMatch = /\[complexity\] REGRESSÃO — (\d+)/.exec(out); - const cycCurrent = cycMatch - ? Number(cycMatch[1]) - : cycOkMatch - ? Number(cycOkMatch[1]) - : cycRegMatch - ? Number(cycRegMatch[1]) - : null; - const cycRegressed = /\[complexity\] REGRESSÃO/.test(out); - record({ - id: "cognitive-complexity", - label: "Cognitive complexity (ratchet)", - kind: "drift", - ok: !cogOver, - detail: - cogCurrent == null - ? "could not parse count" - : `${cogCurrent} vs baseline ${cogBase}${cogOver ? ` (+${cogCurrent - cogBase} drift → rebaseline at release)` : ""}`, - }); - record({ - id: "complexity", - label: "Cyclomatic complexity (ratchet)", - kind: "drift", - ok: !cycRegressed, - detail: - cycCurrent == null - ? firstFailureLine(out) || "measured via check:complexity-ratchets" - : `complexity=${cycCurrent} (shared walk with cognitive)${cycRegressed ? " REGRESSED" : ""}`, - }); + const result = run(npmCmd, ["run", "check:complexity-ratchets"]); + saveGateLog("complexity-ratchets", result.out); + for (const metric of evaluateComplexityRun(result, baselineValue("cognitiveComplexity"))) { + record(metric); + } } // file-size (drift) diff --git a/tests/unit/build/release-green-complexity-outcomes.test.ts b/tests/unit/build/release-green-complexity-outcomes.test.ts new file mode 100644 index 0000000000..e8b0b31d86 --- /dev/null +++ b/tests/unit/build/release-green-complexity-outcomes.test.ts @@ -0,0 +1,105 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + computeVerdict, + evaluateComplexityRun, +} from "../../../scripts/quality/validate-release-green.mjs"; + +test("cancelled complexity measurement cannot emit passing metrics", () => { + const results = evaluateComplexityRun( + { code: 130, out: "Gate execution CANCELLED; see command receipt" }, + 100 + ); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); +}); + +function completeOutput(cognitive = 80, cyclomatic = 20, cyclomaticDrift = false) { + return [ + `complexity=${cyclomatic}`, + `cognitiveComplexity=${cognitive}`, + `[complexity] ${cyclomaticDrift ? "REGRESSÃO" : "OK"} — ${cyclomatic} violações ${cyclomaticDrift ? "> baseline 10" : "(baseline 20)"}`, + `[cognitive-complexity] ${cognitive > 100 ? "REGRESSÃO" : "OK"} — ${cognitive} violações ${cognitive > 100 ? "> baseline 100" : "(baseline 100)"}`, + ].join("\n"); +} + +for (const code of [1, 2, 42, 124, 130, 143]) { + test(`complete metrics cannot hide an unexpected process exit ${code}`, () => { + const results = evaluateComplexityRun({ code, out: completeOutput() }, 100); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); + assert.equal(results[0].kind, "hard"); + }); +} + +for (const out of [ + "", + "complexity=20\n[complexity] OK — 20 violações (baseline 20)", + "cognitiveComplexity=80\n[cognitive-complexity] OK — 80 violações (baseline 100)", + "complexity=20\ncognitiveComplexity=80", + completeOutput().replace("cognitiveComplexity=80", "cognitiveComplexity=NaN"), + completeOutput().replace("complexity=20", "complexity=20.5"), + completeOutput().replace("complexity=20", "complexity=20invalid"), + completeOutput().replace("complexity=20", "complexity=-20"), + completeOutput(Number.MAX_SAFE_INTEGER + 1), +]) { + test(`incomplete or invalid output fails closed: ${JSON.stringify(out)}`, () => { + const results = evaluateComplexityRun({ code: 0, out }, 100); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); + }); +} + +test("unknown baseline fails closed instead of assuming no drift", () => { + for (const baseline of [null, NaN, Infinity, -1]) { + const results = evaluateComplexityRun({ code: 0, out: completeOutput() }, baseline); + assert.equal(computeVerdict(results).releaseGreen, false); + } +}); + +test("completed metrics within baseline retain both passing records", () => { + const results = evaluateComplexityRun({ code: 0, out: completeOutput() }, 100); + assert.deepEqual( + results.map(({ id, kind, ok }) => ({ id, kind, ok })), + [ + { id: "cognitive-complexity", kind: "drift", ok: true }, + { id: "complexity", kind: "drift", ok: true }, + ] + ); + assert.equal(computeVerdict(results).releaseGreen, true); +}); + +for (const [cognitive, cyclomaticDrift] of [ + [101, false], + [80, true], + [101, true], +] as const) { + test(`measured drift remains advisory: cognitive=${cognitive}, cyclomatic=${cyclomaticDrift}`, () => { + const results = evaluateComplexityRun( + { code: 1, out: completeOutput(cognitive, 20, cyclomaticDrift) }, + 100 + ); + assert.equal(results.length, 2); + assert.ok(results.every((result) => result.kind === "drift")); + assert.equal(results[0].ok, cognitive <= 100); + assert.equal(results[1].ok, !cyclomaticDrift); + assert.equal(computeVerdict(results).releaseGreen, true); + }); +} + +test("partial regression output cannot disguise a crashed check as advisory drift", () => { + const results = evaluateComplexityRun( + { code: 1, out: "complexity=20\ncognitiveComplexity=101\nError: incomplete walk" }, + 100 + ); + assert.equal(computeVerdict(results).releaseGreen, false); +});