mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-10-20 05:11:33 +00:00
* fix(ci): fall back to the pinned yamllint when the runner image copy is stale or broken (#12647) Two main-branch CI runs died at `Run yamllint` on the same self-hosted runner (ecs-qwen-hk4-19), each step completing in the same second it started — yamllint never ran. `Install linters` was green because setup trusted `command -v yamllint`: the image ships a copy that is stale or crashes at launch, which satisfied the check, suppressed the pinned install, then died the moment the lane invoked it. A second latent gap made recovery impossible even when the install did run: runCommand appended the pip --user bin dir last in PATH, so the image copy still shadowed the pinned one. Probe the version instead of the name: `yamllint --version` must print exactly the pinned version, or the pinned install runs. And put the pip --user bin dir ahead of the inherited PATH so the pinned install is what the lane resolves. The identical tree lints clean with yamllint 1.35.1, and a same-day run on a healthy runner passed the step in 3s. Coverage: a probe unit test (absent/pinned/stale/crashing copies), a PATH-order unit test, and an end-to-end surrogate replaying the incident — a fake runner image whose yamllint dies on launch, a fake pip3 that installs a healthy stub into the pip --user bin dir, and the real `node scripts/lint.js --setup --yamllint` against the real checkout. Mutation-probed: reverting either half fails the suite. * fix(ci): fail yamllint/shellcheck lanes loudly on an empty git file list (#12647) * refactor(ci): share lint-lane guards and pin lane argv, status, PATH (#12647) Address review on #12650: factor the five git-sourced guard fragments into two parameterized builders (unifying the drifted refusal wording), make getLinterPath's temp dir injectable, probe getLinters() capability instead of gating the seven lane tests on win32, pin both lanes' exact argv and the shellcheck sed severity rewrite, and add a yamllint exit-status propagation test. Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * test(ci): pin getLinterPath defaults and narrow the lane capability probe (#12647) * test(ci): pin the getLinterPath cwd default with a startsWith witness (#12647) --------- Co-authored-by: qwen-code-dev-bot <qwen-code-dev-bot@users.noreply.github.com> Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
398 lines
13 KiB
JavaScript
398 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* @license
|
|
* Copyright 2025 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { execSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdirSync, rmSync } from 'node:fs';
|
|
import { homedir, tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
|
|
const ACTIONLINT_VERSION = '1.7.12';
|
|
const SHELLCHECK_VERSION = '0.11.0';
|
|
const YAMLLINT_VERSION = '1.35.1';
|
|
const ACTIONLINT_SHA256 = {
|
|
linux_amd64:
|
|
'8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8',
|
|
darwin_amd64:
|
|
'5b44c3bc2255115c9b69e30efc0fecdf498fdb63c5d58e17084fd5f16324c644',
|
|
darwin_arm64:
|
|
'aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f',
|
|
};
|
|
const SHELLCHECK_SHA256 = {
|
|
'linux.x86_64':
|
|
'8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198',
|
|
'darwin.x86_64':
|
|
'3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6',
|
|
'darwin.aarch64':
|
|
'56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79',
|
|
};
|
|
|
|
function sanitizePathPart(value) {
|
|
return value.replace(/[^A-Za-z0-9._-]/g, '_');
|
|
}
|
|
|
|
export function getLinterTempDir({
|
|
cwd = process.cwd(),
|
|
env = process.env,
|
|
} = {}) {
|
|
const baseDir = env.RUNNER_TEMP || tmpdir();
|
|
const runId = env.GITHUB_RUN_ID;
|
|
|
|
if (runId) {
|
|
return join(
|
|
baseDir,
|
|
'qwen-code-linters',
|
|
[
|
|
sanitizePathPart(runId),
|
|
sanitizePathPart(env.GITHUB_RUN_ATTEMPT || '1'),
|
|
sanitizePathPart(env.GITHUB_JOB || 'job'),
|
|
].join('-'),
|
|
);
|
|
}
|
|
|
|
const workspaceHash = createHash('sha256')
|
|
.update(cwd)
|
|
.digest('hex')
|
|
.slice(0, 16);
|
|
return join(baseDir, 'qwen-code-linters', `local-${workspaceHash}`);
|
|
}
|
|
|
|
export function getLinterCacheDir({
|
|
env = process.env,
|
|
homeDir = homedir(),
|
|
} = {}) {
|
|
return join(
|
|
env.XDG_CACHE_HOME || join(homeDir, '.cache'),
|
|
'qwen-code',
|
|
'linters',
|
|
);
|
|
}
|
|
|
|
export function getCachedArchiveInstaller({
|
|
cacheArchive,
|
|
localArchive,
|
|
expectedSha256,
|
|
downloadUrl,
|
|
archiveCheck,
|
|
extract,
|
|
executable,
|
|
}) {
|
|
if (!expectedSha256) {
|
|
throw new Error(`Missing SHA-256 pin for ${downloadUrl}`);
|
|
}
|
|
return `
|
|
set -e
|
|
verify_sha256() {
|
|
"${process.execPath}" -e 'const {createHash}=require("node:crypto");const {readFileSync}=require("node:fs");const actual=createHash("sha256").update(readFileSync(process.argv[1])).digest("hex");if(actual!==process.argv[2]){console.error("SHA-256 mismatch for "+process.argv[1]+": expected "+process.argv[2]+", got "+actual);process.exit(1)}' "$1" "$2"
|
|
}
|
|
mkdir -p "${dirname(cacheArchive)}" || true
|
|
if ! cp "${cacheArchive}" "${localArchive}" 2>/dev/null \
|
|
|| ! verify_sha256 "${localArchive}" "${expectedSha256}"; then
|
|
rm -f "${localArchive}"
|
|
curl -fsSL --retry 2 --retry-connrefused --connect-timeout 10 --max-time 90 \
|
|
-o "${localArchive}" "${downloadUrl}"
|
|
verify_sha256 "${localArchive}" "${expectedSha256}"
|
|
${archiveCheck}
|
|
cache_tmp=''
|
|
if cache_tmp="$(mktemp "${cacheArchive}.XXXXXX")" \
|
|
&& cp "${localArchive}" "$cache_tmp" \
|
|
&& "${process.execPath}" -e 'require("node:fs").renameSync(process.argv[1],process.argv[2])' "$cache_tmp" "${cacheArchive}"; then
|
|
:
|
|
else
|
|
[ -z "$cache_tmp" ] || rm -f "$cache_tmp"
|
|
echo "Warning: could not persist linter archive to ${cacheArchive}" >&2
|
|
fi
|
|
fi
|
|
${extract}
|
|
test -x "${executable}"
|
|
`;
|
|
}
|
|
|
|
const TEMP_DIR = getLinterTempDir();
|
|
// Share versioned archives; extracted binaries stay job-scoped in TEMP_DIR.
|
|
const CACHE_DIR = getLinterCacheDir();
|
|
|
|
function getPlatformArch() {
|
|
const platform = process.platform;
|
|
const arch = process.arch;
|
|
if (platform === 'linux' && arch === 'x64') {
|
|
return {
|
|
actionlint: 'linux_amd64',
|
|
shellcheck: 'linux.x86_64',
|
|
};
|
|
}
|
|
if (platform === 'darwin' && arch === 'x64') {
|
|
return {
|
|
actionlint: 'darwin_amd64',
|
|
shellcheck: 'darwin.x86_64',
|
|
};
|
|
}
|
|
if (platform === 'darwin' && arch === 'arm64') {
|
|
return {
|
|
actionlint: 'darwin_arm64',
|
|
shellcheck: 'darwin.aarch64',
|
|
};
|
|
}
|
|
throw new Error(`Unsupported platform/architecture: ${platform}/${arch}`);
|
|
}
|
|
|
|
/**
|
|
* @typedef {{
|
|
* check: string;
|
|
* installer: string;
|
|
* run: string;
|
|
* }}
|
|
*/
|
|
|
|
// Both git-sourced lanes stage the git file list and refuse to hand an empty
|
|
// one to the linter; build the two guard fragments once per stage (the same
|
|
// parameterized-shell pattern as getCachedArchiveInstaller) so the five
|
|
// copies cannot drift apart again.
|
|
const stageGitFileList = (label) => `
|
|
files="$(git ls-files)" || { echo "${label}: git ls-files failed; refusing to lint an empty file list" >&2; exit 1; }`;
|
|
|
|
const refuseEmptyList = (label, variable, reason) => `
|
|
if [ -z "$${variable}" ]; then
|
|
echo "${label}: ${reason}; refusing to pass on an empty file list" >&2
|
|
exit 1
|
|
fi`;
|
|
|
|
let lintersCache;
|
|
|
|
// Built lazily: getPlatformArch() throws on platforms where the POSIX-only
|
|
// linters cannot run (e.g. Windows test hosts importing getLinterTempDir).
|
|
/** @returns {{[linterName: string]: Linter}} */
|
|
export function getLinters() {
|
|
if (!lintersCache) {
|
|
const platformArch = getPlatformArch();
|
|
const actionlintArchive = join(
|
|
CACHE_DIR,
|
|
`actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`,
|
|
);
|
|
const shellcheckArchive = join(
|
|
CACHE_DIR,
|
|
`shellcheck_${SHELLCHECK_VERSION}_${platformArch.shellcheck}.tar.xz`,
|
|
);
|
|
const actionlintLocalArchive = join(TEMP_DIR, '.actionlint.tgz');
|
|
const shellcheckLocalArchive = join(TEMP_DIR, '.shellcheck.txz');
|
|
lintersCache = {
|
|
actionlint: {
|
|
check: 'command -v actionlint',
|
|
installer: `
|
|
mkdir -p "${TEMP_DIR}/actionlint"
|
|
${getCachedArchiveInstaller({
|
|
cacheArchive: actionlintArchive,
|
|
localArchive: actionlintLocalArchive,
|
|
expectedSha256: ACTIONLINT_SHA256[platformArch.actionlint],
|
|
downloadUrl: `https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`,
|
|
archiveCheck: `tar -tzf "${actionlintLocalArchive}" >/dev/null`,
|
|
extract: `tar -xzf "${actionlintLocalArchive}" -C "${TEMP_DIR}/actionlint"`,
|
|
executable: join(TEMP_DIR, 'actionlint', 'actionlint'),
|
|
})}
|
|
`,
|
|
run: `
|
|
actionlint \
|
|
-color \
|
|
-pyflakes= \
|
|
-shellcheck= \
|
|
-ignore 'SC2002:' \
|
|
-ignore 'SC2016:' \
|
|
-ignore 'SC2129:' \
|
|
-ignore 'unexpected key "deployment" for "environment" section' \
|
|
-ignore 'label ".+" is unknown'
|
|
`,
|
|
},
|
|
shellcheck: {
|
|
check: 'command -v shellcheck',
|
|
installer: `
|
|
mkdir -p "${TEMP_DIR}/shellcheck"
|
|
${getCachedArchiveInstaller({
|
|
cacheArchive: shellcheckArchive,
|
|
localArchive: shellcheckLocalArchive,
|
|
expectedSha256: SHELLCHECK_SHA256[platformArch.shellcheck],
|
|
downloadUrl: `https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${platformArch.shellcheck}.tar.xz`,
|
|
archiveCheck: `tar -tf "${shellcheckLocalArchive}" >/dev/null`,
|
|
extract: `tar -xf "${shellcheckLocalArchive}" -C "${TEMP_DIR}/shellcheck" --strip-components=1`,
|
|
executable: join(TEMP_DIR, 'shellcheck', 'shellcheck'),
|
|
})}
|
|
`,
|
|
// Stage the list and refuse to pass on an empty one: the final sed
|
|
// swallows every upstream status, so an empty git ls-files (the
|
|
// 2026-09-24 dubious-ownership failure, #12647) otherwise passed
|
|
// the lane having linted nothing.
|
|
run: `${stageGitFileList('shellcheck')}
|
|
candidates="$(printf '%s\\n' "$files" | grep -v '^integration-tests/terminal-bench/' | grep -E '^([^.]+|.*\\.(sh|zsh|bash))')"${refuseEmptyList(
|
|
'shellcheck',
|
|
'candidates',
|
|
'git ls-files matched no shell-script candidates',
|
|
)}
|
|
scripts="$(printf '%s\\n' "$candidates" | xargs file --mime-type | grep 'text/x-shellscript' | awk '{ print substr($1, 1, length($1)-1) }')"${refuseEmptyList(
|
|
'shellcheck',
|
|
'scripts',
|
|
'file --mime-type detected no shell scripts',
|
|
)}
|
|
printf '%s\\n' "$scripts" | xargs shellcheck \\
|
|
--check-sourced \\
|
|
--enable=all \\
|
|
--exclude=SC2002,SC2129,SC2310 \\
|
|
--severity=style \\
|
|
--format=gcc \\
|
|
--color=never | sed -e 's/note:/warning:/g' -e 's/style:/warning:/g'
|
|
`,
|
|
},
|
|
yamllint: {
|
|
check: 'command -v yamllint',
|
|
installer: `pip3 install --user "yamllint==${YAMLLINT_VERSION}"`,
|
|
// Stage the list and refuse to run on an empty one: when git
|
|
// ls-files dies (the 2026-09-24 dubious-ownership failure, #12647)
|
|
// the lane must fail on git's error, not on yamllint's usage screen
|
|
// from a zero-file invocation — and `xargs -r` would turn that
|
|
// failure into a false green.
|
|
run: `${stageGitFileList('yamllint')}
|
|
files="$(printf '%s\\n' "$files" | grep -E '\\.(yaml|yml)')"${refuseEmptyList(
|
|
'yamllint',
|
|
'files',
|
|
'git ls-files matched no yaml files',
|
|
)}
|
|
printf '%s\\n' "$files" | xargs yamllint --format github
|
|
`,
|
|
},
|
|
};
|
|
}
|
|
return lintersCache;
|
|
}
|
|
|
|
// tempDir defaults to the module TEMP_DIR on purpose: getLinters() builds
|
|
// every installer and extract path from that same constant, so a tempDir
|
|
// that differs from it would put a directory nothing was installed into on
|
|
// PATH.
|
|
export function getLinterPath({
|
|
env = process.env,
|
|
platform = process.platform,
|
|
cwd = process.cwd(),
|
|
tempDir = TEMP_DIR,
|
|
} = {}) {
|
|
const nodeBin = join(cwd, 'node_modules', '.bin');
|
|
let path = `${nodeBin}:${tempDir}/actionlint:${tempDir}/shellcheck:${env.PATH}`;
|
|
if (platform === 'darwin') {
|
|
path = `${path}:${env.HOME}/Library/Python/3.12/bin`;
|
|
} else if (platform === 'linux') {
|
|
path = `${path}:${env.HOME}/.local/bin`;
|
|
}
|
|
return path;
|
|
}
|
|
|
|
function runCommand(command, stdio = 'inherit') {
|
|
try {
|
|
const env = { ...process.env };
|
|
env.PATH = getLinterPath();
|
|
execSync(command, { stdio, env });
|
|
return true;
|
|
} catch (_e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export function setupLinters() {
|
|
console.log('Setting up linters...');
|
|
rmSync(TEMP_DIR, { recursive: true, force: true });
|
|
mkdirSync(TEMP_DIR, { recursive: true });
|
|
|
|
const linters = getLinters();
|
|
for (const linter in linters) {
|
|
const { check, installer } = linters[linter];
|
|
if (!runCommand(check, 'ignore')) {
|
|
console.log(`Installing ${linter}...`);
|
|
if (!runCommand(installer)) {
|
|
console.error(
|
|
`Failed to install ${linter}. Please install it manually.`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
}
|
|
console.log('All required linters are available.');
|
|
}
|
|
|
|
export function runESLint() {
|
|
console.log('\nRunning ESLint...');
|
|
if (!runCommand('npm run lint:ci')) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
export function runActionlint() {
|
|
console.log('\nRunning actionlint...');
|
|
if (!runCommand(getLinters().actionlint.run)) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
export function runShellcheck() {
|
|
console.log('\nRunning shellcheck...');
|
|
if (!runCommand(getLinters().shellcheck.run)) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
export function runYamllint() {
|
|
console.log('\nRunning yamllint...');
|
|
if (!runCommand(getLinters().yamllint.run)) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// `--check`, not `--write`. `--write` reformats in place and exits 0
|
|
// whether or not anything changed, and no caller inspects the tree
|
|
// afterwards, so for the life of this script the Prettier lane reported a
|
|
// pass on unformatted code and CI discarded the rewrites when the job
|
|
// ended. ESLint does not cover the gap either: eslint.config.js loads
|
|
// eslint-config-prettier, which turns formatting rules off rather than
|
|
// enforcing them. `npm run format` remains the way to fix what this
|
|
// reports.
|
|
export function runPrettier() {
|
|
console.log('\nRunning Prettier...');
|
|
if (!runCommand('prettier --experimental-cli --check .')) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
function main() {
|
|
const args = process.argv.slice(2);
|
|
|
|
if (args.includes('--setup')) {
|
|
setupLinters();
|
|
}
|
|
if (args.includes('--eslint')) {
|
|
runESLint();
|
|
}
|
|
if (args.includes('--actionlint')) {
|
|
runActionlint();
|
|
}
|
|
if (args.includes('--shellcheck')) {
|
|
runShellcheck();
|
|
}
|
|
if (args.includes('--yamllint')) {
|
|
runYamllint();
|
|
}
|
|
if (args.includes('--prettier')) {
|
|
runPrettier();
|
|
}
|
|
|
|
if (args.length === 0) {
|
|
setupLinters();
|
|
runESLint();
|
|
runActionlint();
|
|
runShellcheck();
|
|
runYamllint();
|
|
runPrettier();
|
|
console.log('\nAll linting checks passed!');
|
|
}
|
|
}
|
|
|
|
main();
|