test(release): require live channel after restart (#127013)

* test(release): require live channel after restart

* test(release): migrate upgrade channel assertion

* test(release): assert channel runtime after restart

* test(release): await restarted channel runtime

* test(release): prove ClickClack reconnect generation

* test(release): preserve fixture runtime status

---------

Co-authored-by: Amp <amp@ampcode.com>

scripts/e2e/lib/release-upgrade-user-journey/scenario.sh

@@ -173,7 +173,7 @@ node scripts/e2e/lib/release-user-journey/write-clickclack-plugin.mjs "$clickcla
 openclaw plugins install "$clickclack_plugin_dir" --force >"$CLICKCLACK_PLUGIN_INSTALL_LOG" 2>&1
 
 openclaw channels status --json >"$STATUS_JSON" 2>"$STATUS_ERR"
-node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-status clickclack "$STATUS_JSON"
+node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-configured clickclack "$STATUS_JSON"
 openclaw message send \
   --channel clickclack \
   --target channel:general \

scripts/e2e/lib/release-user-journey/assertions.mjs

@@ -277,18 +277,31 @@ function configureClickClack() {
   writeConfig(cfg);
 }
 
-function assertChannelStatus() {
+function readChannelStatus() {
   const channel = process.argv[3];
   const statusPath = process.argv[4];
   const status = readJson(statusPath);
+  return { channel, status };
+}
+
+function assertChannelConfigured() {
+  const { channel, status } = readChannelStatus();
   const configured = Array.isArray(status.configuredChannels) ? status.configuredChannels : [];
-  const liveStatus = status.channels?.[channel];
   assert(
-    configured.includes(channel) || liveStatus?.ok === true,
-    `${channel} missing from channels status: ${JSON.stringify(status)}`,
+    configured.includes(channel),
+    `${channel} missing from configured channels: ${JSON.stringify(status)}`,
   );
 }
 
+function assertChannelRunning() {
+  const { channel, status } = readChannelStatus();
+  const accounts = status.channelAccounts?.[channel];
+  const defaultAccount = Array.isArray(accounts)
+    ? accounts.find((account) => account?.accountId === "default")
+    : undefined;
+  assert(defaultAccount?.running === true, `${channel} is not running: ${JSON.stringify(status)}`);
+}
+
 async function postClickClackInbound() {
   const baseUrl = process.argv[3];
   const body = process.argv[4];
@@ -315,10 +328,24 @@ async function waitClickClackSocket() {
     30,
     "ClickClack websocket timeout seconds",
   );
-  await waitForClickClackSocket({ baseUrl, timeoutMs: timeoutSeconds * 1000 });
+  const minimumSocketGeneration = readPositiveInt(
+    process.argv[5],
+    1,
+    "ClickClack minimum websocket generation",
+  );
+  await waitForClickClackSocket({
+    baseUrl,
+    timeoutMs: timeoutSeconds * 1000,
+    minimumSocketGeneration,
+  });
 }
 
-export async function waitForClickClackSocket({ baseUrl, timeoutMs, pollIntervalMs = 250 }) {
+export async function waitForClickClackSocket({
+  baseUrl,
+  timeoutMs,
+  minimumSocketGeneration = 1,
+  pollIntervalMs = 250,
+}) {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
     const remainingMs = Math.max(1, deadline - Date.now());
@@ -334,7 +361,7 @@ export async function waitForClickClackSocket({ baseUrl, timeoutMs, pollInterval
       },
     ).catch(() => undefined);
     if (state) {
-      if (Number(state.socketCount ?? 0) > 0) {
+      if (Number(state.socketGeneration ?? 0) >= minimumSocketGeneration) {
         return;
       }
     }
@@ -342,7 +369,9 @@ export async function waitForClickClackSocket({ baseUrl, timeoutMs, pollInterval
       setTimeout(resolve, Math.min(pollIntervalMs, Math.max(0, deadline - Date.now())));
     });
   }
-  throw new Error(`Timed out waiting for ClickClack websocket connection at ${baseUrl}`);
+  throw new Error(
+    `Timed out waiting for ClickClack websocket generation ${minimumSocketGeneration} at ${baseUrl}`,
+  );
 }
 
 function assertClickClackState() {
@@ -382,7 +411,8 @@ const commands = {
   "assert-file-contains": assertFileContains,
   "assert-plugin-uninstalled": assertPluginUninstalled,
   "configure-clickclack": configureClickClack,
-  "assert-channel-status": assertChannelStatus,
+  "assert-channel-configured": assertChannelConfigured,
+  "assert-channel-running": assertChannelRunning,
   "post-clickclack-inbound": postClickClackInbound,
   "wait-clickclack-socket": waitClickClackSocket,
   "assert-clickclack-state": assertClickClackState,

scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs

@@ -44,6 +44,7 @@ const messages = [];
 const threadReplies = [];
 const outboundMessages = [];
 const sockets = new Set();
+let socketGeneration = 0;
 
 function persist() {
   fs.writeFileSync(
@@ -54,6 +55,7 @@ function persist() {
         threadReplies,
         outboundMessages,
         socketCount: sockets.size,
+        socketGeneration,
       },
       null,
       2,
@@ -289,7 +291,13 @@ async function handleRequest(req, res) {
       return;
     }
     if (req.method === "GET" && url.pathname === "/fixture/state") {
-      json(res, 200, { messages, threadReplies, outboundMessages, socketCount: sockets.size });
+      json(res, 200, {
+        messages,
+        threadReplies,
+        outboundMessages,
+        socketCount: sockets.size,
+        socketGeneration,
+      });
       return;
     }
     json(res, 404, { error: `unhandled ${req.method} ${url.pathname}` });
@@ -328,15 +336,20 @@ server.on("upgrade", (req, socket) => {
     ].join("\r\n"),
   );
   sockets.add(socket);
+  socketGeneration += 1;
   persist();
-  socket.on("close", () => {
-    sockets.delete(socket);
-    persist();
-  });
-  socket.on("error", () => {
-    sockets.delete(socket);
-    persist();
+  const cleanupSocket = () => {
+    if (sockets.delete(socket)) {
+      persist();
+    }
+  };
+  socket.on("close", cleanupSocket);
+  socket.on("end", () => {
+    cleanupSocket();
+    socket.end();
   });
+  socket.on("error", cleanupSocket);
+  socket.resume();
 });
 
 persist();

scripts/e2e/lib/release-user-journey/scenario.sh

@@ -225,7 +225,7 @@ openclaw plugins install "$clickclack_plugin_dir" --force >"$CLICKCLACK_PLUGIN_I
 echo "Configuring ClickClack..."
 node scripts/e2e/lib/release-user-journey/assertions.mjs configure-clickclack "http://127.0.0.1:$CLICKCLACK_PORT"
 openclaw channels status --json >"$STATUS_JSON" 2>"$STATUS_ERR"
-node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-status clickclack "$STATUS_JSON"
+node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-configured clickclack "$STATUS_JSON"
 
 echo "Sending ClickClack outbound message..."
 openclaw message send \
@@ -244,9 +244,10 @@ node scripts/e2e/lib/release-user-journey/assertions.mjs wait-clickclack-reply "
 echo "Restarting Gateway and checking state survival..."
 stop_gateway
 start_gateway "$GATEWAY_2_LOG"
+node scripts/e2e/lib/release-user-journey/assertions.mjs wait-clickclack-socket "http://127.0.0.1:$CLICKCLACK_PORT" 45 2
 openclaw plugins inspect journey-plugin-b --runtime --json >"$PLUGIN_B_AFTER_RESTART_JSON" 2>&1
 openclaw channels status --json >"$STATUS_AFTER_RESTART_JSON" 2>"$STATUS_AFTER_RESTART_ERR"
-node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-status clickclack "$STATUS_AFTER_RESTART_JSON"
+node scripts/e2e/lib/release-user-journey/assertions.mjs assert-channel-running clickclack "$STATUS_AFTER_RESTART_JSON"
 node scripts/e2e/lib/release-user-journey/assertions.mjs assert-file-contains "$PLUGIN_B_AFTER_RESTART_JSON" "journey-plugin-b"
 stop_gateway
 

scripts/e2e/lib/release-user-journey/write-clickclack-plugin.mjs

@@ -362,11 +362,15 @@ const clickclackPlugin = {
       label: snapshot.configured ? "configured" : "missing config",
       detail: snapshot.baseUrl ?? "",
     }),
-    buildAccountSnapshot: ({ account }) => ({
+    buildAccountSnapshot: ({ account, runtime }) => ({
       accountId: account.accountId,
       enabled: account.enabled,
       configured: account.configured,
       baseUrl: account.baseUrl,
+      running: runtime?.running ?? false,
+      lastStartAt: runtime?.lastStartAt ?? null,
+      lastStopAt: runtime?.lastStopAt ?? null,
+      lastError: runtime?.lastError ?? null,
     }),
   },
   outbound: {

test/scripts/release-user-journey-assertions.test.ts

@@ -1,18 +1,26 @@
 // Release User Journey Assertions tests cover release user journey assertions script behavior.
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync, type ChildProcess } from "node:child_process";
+import { once } from "node:events";
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { createServer, type AddressInfo, type Socket } from "node:net";
 import { tmpdir } from "node:os";
 import path from "node:path";
-import { describe, expect, it } from "vitest";
+import { pathToFileURL } from "node:url";
+import { afterEach, describe, expect, it } from "vitest";
+import { WebSocket } from "ws";
 import {
   runReleaseUserJourneyAssertion,
   waitForClickClackSocket,
 } from "../../scripts/e2e/lib/release-user-journey/assertions.mjs";
 import { withEnvAsync } from "../../src/test-utils/env.js";
+import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
 
 const ASSERTIONS_SCRIPT = "scripts/e2e/lib/release-user-journey/assertions.mjs";
+const CLICKCLACK_FIXTURE_SCRIPT = "scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs";
+const CLICKCLACK_PLUGIN_WRITER_SCRIPT =
+  "scripts/e2e/lib/release-user-journey/write-clickclack-plugin.mjs";
 const DISABLE_EXPERIMENTAL_WARNING = "--disable-warning=ExperimentalWarning";
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
 
 function nodeOptionsWithoutExperimentalWarnings(extra?: string): string {
   const current = [process.env.NODE_OPTIONS, extra].filter(Boolean).join(" ");
@@ -44,10 +52,10 @@ function runAssertion(
   });
 }
 
-async function waitUntil(matches: () => boolean, label: string): Promise<void> {
+async function waitUntil(matches: () => boolean | Promise<boolean>, label: string): Promise<void> {
   const startedAt = Date.now();
   while (Date.now() - startedAt < 1000) {
-    if (matches()) {
+    if (await matches()) {
       return;
     }
     await new Promise((resolve) => {
@@ -57,6 +65,37 @@ async function waitUntil(matches: () => boolean, label: string): Promise<void> {
   throw new Error(`timed out waiting for ${label}`);
 }
 
+async function reserveTcpPort(): Promise<number> {
+  const server = createServer();
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const port = (server.address() as AddressInfo).port;
+  await new Promise<void>((resolve, reject) => {
+    server.close((error) => (error ? reject(error) : resolve()));
+  });
+  return port;
+}
+
+async function stopChild(child: ChildProcess): Promise<void> {
+  if (child.exitCode !== null) {
+    return;
+  }
+  child.kill("SIGTERM");
+  await once(child, "exit");
+}
+
+async function openClickClackSocket(port: number, token: string): Promise<WebSocket> {
+  const socket = new WebSocket(`ws://127.0.0.1:${port}/api/realtime/ws`, {
+    headers: { authorization: `Bearer ${token}` },
+  });
+  await once(socket, "open");
+  return socket;
+}
+
+async function closeClickClackSocket(socket: WebSocket): Promise<void> {
+  socket.terminate();
+  await once(socket, "close");
+}
+
 async function startTcpFixtureServer(handler: (socket: Socket) => void): Promise<{
   port: number;
   stop: () => Promise<void>;
@@ -175,6 +214,51 @@ describe("release user journey assertions", () => {
     }
   });
 
+  it("accepts a configured channel before Gateway startup", async () => {
+    const root = tempDirs.make("openclaw-release-user-assertions-");
+    const statusPath = path.join(root, "status.json");
+
+    writeJson(statusPath, { configuredChannels: ["clickclack"] });
+
+    await expect(
+      runReleaseUserJourneyAssertion("assert-channel-configured", ["clickclack", statusPath]),
+    ).resolves.toBeUndefined();
+  });
+
+  it("rejects a configured channel that failed after Gateway restart", async () => {
+    const root = tempDirs.make("openclaw-release-user-assertions-");
+    const statusPath = path.join(root, "status.json");
+
+    writeJson(statusPath, {
+      configuredChannels: ["clickclack"],
+      channels: { clickclack: { ok: true, label: "configured" } },
+      channelAccounts: {
+        clickclack: [{ accountId: "default", configured: true, running: false }],
+      },
+    });
+
+    await expect(
+      runReleaseUserJourneyAssertion("assert-channel-running", ["clickclack", statusPath]),
+    ).rejects.toThrow("clickclack is not running");
+  });
+
+  it("accepts a running channel after Gateway restart", async () => {
+    const root = tempDirs.make("openclaw-release-user-assertions-");
+    const statusPath = path.join(root, "status.json");
+
+    writeJson(statusPath, {
+      configuredChannels: ["clickclack"],
+      channels: { clickclack: { ok: true } },
+      channelAccounts: {
+        clickclack: [{ accountId: "default", configured: true, running: true }],
+      },
+    });
+
+    await expect(
+      runReleaseUserJourneyAssertion("assert-channel-running", ["clickclack", statusPath]),
+    ).resolves.toBeUndefined();
+  });
+
   it("fails when uninstall leaves the managed plugin directory behind", () => {
     const root = mkdtempSync(path.join(tmpdir(), "openclaw-release-user-assertions-"));
     const home = path.join(root, "home");
@@ -276,31 +360,133 @@ describe("release user journey assertions", () => {
     }
   });
 
-  it("accepts ready ClickClack fixture state", async () => {
-    const root = mkdtempSync(path.join(tmpdir(), "openclaw-release-user-assertions-"));
-    const home = path.join(root, "home");
-    const server = await startTcpFixtureServer((socket) => {
-      const body = JSON.stringify({ socketCount: 1 });
-      socket.end(
-        `HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`,
-      );
+  it("waits for a new ClickClack websocket generation across reconnect", async () => {
+    const root = tempDirs.make("openclaw-release-user-assertions-");
+    const statePath = path.join(root, "clickclack.json");
+    const port = await reserveTcpPort();
+    const token = "clickclack-test-token";
+    const fixture = spawn(process.execPath, [CLICKCLACK_FIXTURE_SCRIPT], {
+      env: {
+        ...process.env,
+        CLICKCLACK_FIXTURE_PORT: String(port),
+        CLICKCLACK_FIXTURE_STATE: statePath,
+        CLICKCLACK_FIXTURE_TOKEN: token,
+      },
+      stdio: "ignore",
     });
+    let socket: WebSocket | undefined;
 
     try {
-      await expect(
-        withEnvAsync({ HOME: home, OPENCLAW_RELEASE_USER_JOURNEY_HTTP_TIMEOUT_MS: "1000" }, () =>
-          runReleaseUserJourneyAssertion("wait-clickclack-socket", [
-            `http://127.0.0.1:${server.port}`,
-            "1",
-          ]),
+      await waitUntil(
+        async () =>
+          fetch(`http://127.0.0.1:${port}/health`)
+            .then((response) => response.ok)
+            .catch(() => false),
+        "ClickClack fixture startup",
+      );
+
+      socket = await openClickClackSocket(port, token);
+      await waitForClickClackSocket({
+        baseUrl: `http://127.0.0.1:${port}`,
+        pollIntervalMs: 20,
+        timeoutMs: 1000,
+      });
+      await closeClickClackSocket(socket);
+      socket = undefined;
+      await waitUntil(async () => {
+        const state = JSON.parse(
+          await fetch(`http://127.0.0.1:${port}/fixture/state`).then((response) => response.text()),
+        );
+        return state.socketCount === 0 && state.socketGeneration === 1;
+      }, "first ClickClack socket close");
+
+      const secondGeneration = waitForClickClackSocket({
+        baseUrl: `http://127.0.0.1:${port}`,
+        minimumSocketGeneration: 2,
+        pollIntervalMs: 20,
+        timeoutMs: 1000,
+      });
+      socket = await openClickClackSocket(port, token);
+      await expect(secondGeneration).resolves.toBeUndefined();
+      expect(
+        JSON.parse(
+          await fetch(`http://127.0.0.1:${port}/fixture/state`).then((response) => response.text()),
         ),
-      ).resolves.toBeUndefined();
+      ).toMatchObject({ socketCount: 1, socketGeneration: 2 });
     } finally {
-      await server.stop();
-      rmSync(root, { force: true, recursive: true });
+      if (socket) {
+        await closeClickClackSocket(socket);
+      }
+      await stopChild(fixture);
     }
   });
 
+  it("preserves runtime state in generated ClickClack status snapshots", async () => {
+    type GeneratedClickClackPlugin = {
+      status: {
+        buildAccountSnapshot: (params: {
+          account: { accountId: string; enabled: boolean; configured: boolean; baseUrl: string };
+          runtime?: {
+            running?: boolean;
+            lastStartAt?: number;
+            lastStopAt?: number;
+            lastError?: string;
+          };
+        }) => Record<string, unknown>;
+      };
+    };
+
+    const pluginDir = path.join(tempDirs.make("openclaw-release-clickclack-plugin-"), "plugin");
+    const writer = spawnSync(process.execPath, [CLICKCLACK_PLUGIN_WRITER_SCRIPT, pluginDir], {
+      encoding: "utf8",
+    });
+    expect(writer.status, writer.stderr).toBe(0);
+
+    const generatedModule = (await import(
+      pathToFileURL(path.join(pluginDir, "index.mjs")).href
+    )) as {
+      default: {
+        register: (api: {
+          registerChannel: (registration: { plugin: GeneratedClickClackPlugin }) => void;
+        }) => void;
+      };
+    };
+    let plugin: GeneratedClickClackPlugin | undefined;
+    generatedModule.default.register({
+      registerChannel: (registration) => {
+        plugin = registration.plugin;
+      },
+    });
+    if (!plugin) {
+      throw new Error("generated ClickClack plugin did not register its channel");
+    }
+
+    expect(
+      plugin.status.buildAccountSnapshot({
+        account: {
+          accountId: "default",
+          enabled: true,
+          configured: true,
+          baseUrl: "http://127.0.0.1:1234",
+        },
+        runtime: {
+          running: true,
+          lastStartAt: 123,
+          lastStopAt: 45,
+          lastError: "prior disconnect",
+        },
+      }),
+    ).toMatchObject({
+      accountId: "default",
+      configured: true,
+      enabled: true,
+      running: true,
+      lastStartAt: 123,
+      lastStopAt: 45,
+      lastError: "prior disconnect",
+    });
+  });
+
   it("cancels successful ClickClack inbound response bodies", async () => {
     const root = mkdtempSync(path.join(tmpdir(), "openclaw-release-user-assertions-"));
     const home = path.join(root, "home");
@@ -345,7 +531,7 @@ describe("release user journey assertions", () => {
             timeoutMs: 150,
           }),
         ),
-      ).rejects.toThrow("Timed out waiting for ClickClack websocket connection");
+      ).rejects.toThrow("Timed out waiting for ClickClack websocket generation 1");
       expect(Date.now() - startedAt).toBeLessThan(750);
     } finally {
       await server.stop();