mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-10-22 06:49:52 +00:00
test(integration): recognize sealed writer renewals in crash gate (#13385)
* test(integration): recognize sealed writer renewals * test(integration): tolerate renewals from a killed writer grant The crash gate's 409 tolerance was keyed only on a grant the proxy watched seal. On the worker-stop path the harness is SIGKILLed before it can seal, so the fenced renewal recorded in #13370 still fell through to the strict branch and failed the gate identically. Key the tolerance on the union of both fence signals, while never accepting the current boot's grant. Also pin the production invariant that tolerance relies on: a post-seal /writers:renew rejection stays swallowed by the renewal timer. The core fake store now fences a renewal arriving after its writer sealed. Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> Patrol-Run: qwen-pr-closeout/jmutqht184t --------- Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
parent
5022712321
commit
ffc9772423
2 changed files with 87 additions and 1 deletions
|
|
@ -38,6 +38,7 @@ const report = {
|
|||
operations: [] as string[],
|
||||
signalEvidence: [] as unknown[],
|
||||
restoreTransactions: [] as Transaction[],
|
||||
fencedWriterRenewals: 0,
|
||||
target: undefined as Transaction | undefined,
|
||||
};
|
||||
const proof = path.join(config.directory, 'proof.txt');
|
||||
|
|
@ -49,6 +50,8 @@ let restoring = false;
|
|||
let injected = false;
|
||||
let serviceKilled = false;
|
||||
let proxyFailure: unknown;
|
||||
const sealedWriters = new Set<string>();
|
||||
const killedWriters = new Set<string>();
|
||||
|
||||
function events(transaction: Transaction) {
|
||||
return Buffer.from(transaction.recordBytesBase64, 'base64')
|
||||
|
|
@ -79,6 +82,7 @@ async function killHarness() {
|
|||
assert.equal(code, null);
|
||||
assert.equal(signal, 'SIGKILL');
|
||||
report.signalEvidence.push({ process: 'harness', pid, signal });
|
||||
killedWriters.add(cli.bootId);
|
||||
}
|
||||
|
||||
async function enteredTool() {
|
||||
|
|
@ -153,8 +157,33 @@ const proxy = createServer(async (req, res) => {
|
|||
?.includes('application/json')
|
||||
? JSON.parse(bytes.toString())
|
||||
: undefined;
|
||||
const writerGrant = `${fields.writerId}:${fields.writerGeneration}`;
|
||||
if (
|
||||
store &&
|
||||
url.pathname.endsWith('/writers:seal') &&
|
||||
upstream.status === 200
|
||||
)
|
||||
sealedWriters.add(writerGrant);
|
||||
if (!serviceKilled) {
|
||||
if (upstream.status === 409) {
|
||||
if (
|
||||
upstream.status === 409 &&
|
||||
store &&
|
||||
url.pathname.endsWith('/writers:renew') &&
|
||||
(sealedWriters.has(writerGrant) ||
|
||||
(killedWriters.has(fields.writerId) &&
|
||||
fields.writerId !== cli.bootId))
|
||||
) {
|
||||
// A renewal already in flight may reach the Store after its writer
|
||||
// sealed, or after the driver SIGKILLed the harness holding it. The
|
||||
// current boot's grant is never tolerated: catching that conflict is
|
||||
// the whole reason the fence exists.
|
||||
assert.equal(
|
||||
json.error.code,
|
||||
'managed_session_writer_conflict',
|
||||
`${url}: ${bytes}`,
|
||||
);
|
||||
report.fencedWriterRenewals++;
|
||||
} else if (upstream.status === 409) {
|
||||
assert(
|
||||
injected &&
|
||||
!store &&
|
||||
|
|
|
|||
|
|
@ -953,6 +953,52 @@ describe('HTTP Managed Session store', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('swallows a renewal the Store fences after its writer sealed', async () => {
|
||||
vi.useFakeTimers();
|
||||
const server = new FakeManagedSessionStore();
|
||||
let finishRenewal!: () => void;
|
||||
const renewalGate = new Promise<void>((resolve) => {
|
||||
finishRenewal = resolve;
|
||||
});
|
||||
let fencedRenewals = 0;
|
||||
const fetchFn = vi.fn<typeof fetch>(async (input, init) => {
|
||||
const renewal = requestUrl(input).endsWith('/writers:renew');
|
||||
if (renewal) await renewalGate;
|
||||
const response = await server.fetch(input, init);
|
||||
if (renewal && response.status === 409) fencedRenewals++;
|
||||
return response;
|
||||
});
|
||||
const stores = createHttpManagedSessionStores({
|
||||
baseUrl: 'http://session-store.test',
|
||||
sessionKey: SESSION_KEY,
|
||||
writerId: 'harness-a',
|
||||
writerToken: TOKEN_A,
|
||||
leaseDurationMs: 1000,
|
||||
fetchFn,
|
||||
});
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandledRejection = (reason: unknown) => {
|
||||
unhandled.push(reason);
|
||||
};
|
||||
process.on('unhandledRejection', onUnhandledRejection);
|
||||
try {
|
||||
await stores.journalStore.open({ sessionKey: SESSION_KEY });
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
const closing = stores.close();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
finishRenewal();
|
||||
await closing;
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(fencedRenewals).toBe(1);
|
||||
expect(unhandled).toHaveLength(0);
|
||||
} finally {
|
||||
process.off('unhandledRejection', onUnhandledRejection);
|
||||
finishRenewal();
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('commits the resources a Stage H record names and rebuilds it cold', async () => {
|
||||
const server = new FakeManagedSessionStore();
|
||||
const runtimeBaseDir = await mkdtemp(
|
||||
|
|
@ -1420,6 +1466,17 @@ class FakeManagedSessionStore {
|
|||
return jsonResponse(this.grant());
|
||||
}
|
||||
if (suffix === '/writers:renew') {
|
||||
if (this.state === 'SEALED')
|
||||
return jsonResponse(
|
||||
{
|
||||
error: {
|
||||
code: 'managed_session_writer_conflict',
|
||||
message:
|
||||
'The Managed Session writer grant is stale or unavailable.',
|
||||
},
|
||||
},
|
||||
409,
|
||||
);
|
||||
this.leaseUntil = Date.now() + 300_000;
|
||||
return jsonResponse(this.grant());
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue