vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-11-01 21:18:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 29

fix(docker): preserve frozen workspace installs

Browse source 
Stage every workspace manifest required by pnpm 12.7 frozen-lockfile
validation, including optional plugins and examples. Filter production
installation through the existing required and selected plugin list.
Keep the full dependency graph in the disposable build stage for source
asset preparation, then retain the existing runtime pruning.

Execute the Docker manifest extraction in the regression guard and check
all lockfile importers, with default and Matrix selections.

Refs #167923
This commit is contained in:
Peter Steinberger 2026-10-09 13:43:22 -07:00
parent d0fc39e872
commit b7e788b23e
No known key found for this signature in database
2 changed files with 158 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

34
Dockerfile
View file

@ -30,7 +30,8 @@ ARG OPENCLAW_BUN_IMAGE="docker.io/oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5
FROM ${OPENCLAW_NODE_BOOKWORM_IMAGE} AS workspace-deps
ARG OPENCLAW_EXTENSIONS
ARG OPENCLAW_BUNDLED_PLUGIN_DIR
# Copy package.json files for workspace packages used by the install layer.
# Frozen installs validate every lockfile importer's manifest, including unselected plugins.
# Stage all workspace manifests, then filter dependency installation separately.
# Manifest-only bundled plugins remain valid selections but need no workspace metadata.
# Use COPY because build-context bind mounts are unreliable across supported
# Podman/Buildah hosts. Full trees stay in this disposable stage; later stages
@ -39,28 +40,21 @@ COPY scripts/lib/docker-plugin-selection.mjs /tmp/docker-plugin-selection.mjs
COPY scripts/lib/root-package-bundled-plugin-excludes.mjs /tmp/root-package-bundled-plugin-excludes.mjs
COPY package.json /tmp/package.json
COPY packages /tmp/packages
COPY examples /tmp/examples
COPY ${OPENCLAW_BUNDLED_PLUGIN_DIR} /tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}
RUN mkdir -p /out/packages "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}" && \
for manifest in /tmp/packages/*/package.json; do \
RUN mkdir -p /out/packages /out/examples "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}" && \
for manifest in /tmp/packages/*/package.json /tmp/examples/*/package.json "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}"/*/package.json; do \
[ -f "$manifest" ] || continue; \
pkg_dir="${manifest%/package.json}"; \
pkg_name="${pkg_dir##*/}"; \
mkdir -p "/out/packages/$pkg_name" && \
cp "$manifest" "/out/packages/$pkg_name/package.json"; \
manifest_path="${manifest#/tmp/}"; \
mkdir -p "/out/${manifest_path%/package.json}" && \
cp "$manifest" "/out/$manifest_path"; \
done && \
node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS" \
> /out/openclaw-selected-plugin-dirs && \
node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS" \
--required-platform-packages > /out/openclaw-required-platform-packages && \
node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS" \
--required-bundled /tmp/package.json > /tmp/openclaw-workspace-plugin-dirs && \
while IFS= read -r ext; do \
ext_dir="/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext"; \
if [ -f "$ext_dir/package.json" ]; then \
mkdir -p "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext" && \
cp "$ext_dir/package.json" "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json"; \
fi; \
done < /tmp/openclaw-workspace-plugin-dirs
--required-bundled /tmp/package.json > /out/openclaw-workspace-plugin-dirs
# Shared manifest-only inputs. Both installs start without node_modules so pnpm
# never has to rename a dependency directory inherited from an OverlayFS layer.
@ -93,14 +87,20 @@ COPY scripts/docker/verify-fs-safe-native.mjs ./scripts/docker/verify-fs-safe-na
COPY scripts/docker/verify-native-addons.sh ./scripts/docker/verify-native-addons.sh
COPY --from=workspace-deps /out/packages/ ./packages/
COPY --from=workspace-deps /out/examples/ ./examples/
COPY --from=workspace-deps /out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/ ./${OPENCLAW_BUNDLED_PLUGIN_DIR}/
COPY --from=workspace-deps /out/openclaw-workspace-plugin-dirs /tmp/openclaw-workspace-plugin-dirs
COPY --from=workspace-deps /out/openclaw-selected-plugin-dirs /tmp/openclaw-selected-plugin-dirs
COPY --from=workspace-deps /out/openclaw-required-platform-packages /tmp/openclaw-required-platform-packages
# ── Production dependencies ────────────────────────────────────
FROM dependency-inputs AS production-deps
RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/store,sharing=locked \
NODE_OPTIONS=--max-old-space-size=2048 pnpm install --frozen-lockfile --prod \
set -eu; set -- --filter . --filter ./ui --filter './packages/*'; \
while IFS= read -r ext; do \
set -- "$@" --filter "./${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext"; \
done < /tmp/openclaw-workspace-plugin-dirs; \
NODE_OPTIONS=--max-old-space-size=2048 pnpm install --frozen-lockfile --prod "$@" \
--config.supportedArchitectures.os=linux \
--config.supportedArchitectures.cpu="$(node -p 'process.arch')" \
--config.supportedArchitectures.libc=glibc
@ -120,6 +120,8 @@ COPY --from=bun-binary /usr/local/bin/bun /usr/local/bin/bun
# Reduce OOM risk on low-memory hosts during dependency installation.
# Docker builds on small VMs may otherwise fail with "Killed" (exit 137).
# Source asset preparation builds external plugins before runtime pruning.
# Its build dependencies stay in this disposable stage; production remains filtered.
RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/store,sharing=locked \
NODE_OPTIONS=--max-old-space-size=2048 pnpm install --frozen-lockfile \
--config.supportedArchitectures.os=linux \

147
src/dockerfile.test.ts
View file

@ -1,13 +1,26 @@
// Tests Dockerfile metadata and expected install commands.
import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { access, cp, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
import {
access,
cp,
mkdir,
mkdtemp,
readFile,
readdir,
rm,
stat,
symlink,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, join, posix, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { BUNDLED_PLUGIN_ROOT_DIR } from "openclaw/plugin-sdk/test-fixtures";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { isMap, parseAllDocuments } from "yaml";
import { collectPackageDistImportErrors } from "../scripts/lib/package-dist-imports.mjs";
import { useAutoCleanupTempDirTracker } from "../test/helpers/temp-dir.js";
import { resolveTestNodeExecPath } from "./test-utils/node-process.js";
const repoRoot = resolve(fileURLToPath(new URL(".", import.meta.url)), "..");
@ -16,11 +29,57 @@ const dockerComposePath = join(repoRoot, "docker-compose.yml");
const dockerInstallDocsPath = join(repoRoot, "docs/install/docker.md");
const composeSetupScriptPath = join(repoRoot, "scripts/e2e/compose-setup.sh");
const dockerSetupDockerfilePaths = ["Dockerfile", "scripts/docker/sandbox/Dockerfile"] as const;
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
function collapseDockerContinuations(dockerfile: string): string {
return dockerfile.replace(/\\\r?\n[ \t]*/g, " ");
}
async function stageDockerManifestCopies(params: { body: string; fixture: string; stage: string }) {
const stageRoot = join(params.fixture, params.stage);
const body = params.body.replaceAll("${OPENCLAW_BUNDLED_PLUGIN_DIR}", BUNDLED_PLUGIN_ROOT_DIR);
for (const [, from, inputs, destination] of body.matchAll(
/^COPY (?:--from=(\S+) )?(.+) (\S+)$/gm,
)) {
if (!inputs || !destination) {
throw new Error("Docker manifest COPY inputs are missing");
}
const sourceRoot = from ? join(params.fixture, from) : repoRoot;
const destinationRoot = destination.startsWith("/")
? join(stageRoot, destination)
: join(stageRoot, "app", destination);
for (const input of inputs.split(/\s+/)) {
const source = join(sourceRoot, input);
const sourceStat = await stat(source);
const target =
!sourceStat.isDirectory() && destination.endsWith("/")
? join(destinationRoot, basename(input))
: destinationRoot;
await mkdir(dirname(target), { recursive: true });
if (from || sourceStat.isFile()) {
await cp(source, target, { recursive: true });
continue;
}
await mkdir(target, { recursive: true });
// The extraction stage reads only direct workspace manifests, not their source trees.
for (const entry of await readdir(source, { withFileTypes: true })) {
if (!entry.isDirectory()) {
continue;
}
for (const file of await readdir(join(source, entry.name))) {
if (file !== "package.json" && file !== "openclaw.plugin.json") {
continue;
}
const manifestTarget = join(target, entry.name, file);
await mkdir(dirname(manifestTarget), { recursive: true });
await cp(join(source, entry.name, file), manifestTarget);
}
}
}
}
return stageRoot;
}
function resolveOptionalAptPackages(dockerfile: string, env: NodeJS.ProcessEnv): string {
const assignment = collapseDockerContinuations(dockerfile).match(
/\bpackages="(\$\{OPENCLAW_IMAGE_APT_PACKAGES:-\$OPENCLAW_DOCKER_APT_PACKAGES\})";/u,
@ -207,12 +266,11 @@ describe("Dockerfile", () => {
expect(workspaceDepsEnd).toBeGreaterThan(workspaceDepsStart);
const workspaceDeps = dockerfile.slice(workspaceDepsStart, workspaceDepsEnd);
const extractionIndex = workspaceDeps.indexOf(
'RUN mkdir -p /out/packages "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}"',
);
const extractionIndex = workspaceDeps.indexOf("RUN mkdir -p /out/");
const inputCopies = [
"COPY scripts/lib/docker-plugin-selection.mjs /tmp/docker-plugin-selection.mjs",
"COPY packages /tmp/packages",
"COPY examples /tmp/examples",
"COPY ${OPENCLAW_BUNDLED_PLUGIN_DIR} /tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}",
];
@ -239,18 +297,23 @@ describe("Dockerfile", () => {
const extensionManifestIndex = dockerfile.indexOf(
"COPY --from=workspace-deps /out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/ ./${OPENCLAW_BUNDLED_PLUGIN_DIR}/",
);
const exampleManifestIndex = dockerfile.indexOf(
"COPY --from=workspace-deps /out/examples/ ./examples/",
);
expect(postinstallIndex).toBeGreaterThan(-1);
expect(prepareIndex).toBeGreaterThan(-1);
expect(distImportHelperIndex).toBeGreaterThan(-1);
expect(packageManifestIndex).toBeGreaterThan(-1);
expect(extensionManifestIndex).toBeGreaterThan(-1);
expect(exampleManifestIndex).toBeGreaterThan(-1);
expect(dockerfile).toContain("for manifest in /tmp/packages/*/package.json");
expect(dockerfile).toContain(
'node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS"',
);
expect(dockerfile).toContain("done < /tmp/openclaw-workspace-plugin-dirs");
expect(dockerfile).toContain(`if [ -f "$ext_dir/package.json" ]; then`);
expect(dockerfile).toContain("/tmp/examples/*/package.json");
expect(dockerfile).toContain('"/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}"/*/package.json');
expect(dockerfile).toContain('[ -f "$manifest" ] || continue');
expect(dockerfile).toContain(
"COPY --from=workspace-deps /out/openclaw-selected-plugin-dirs /tmp/openclaw-selected-plugin-dirs",
);
@ -259,8 +322,78 @@ describe("Dockerfile", () => {
expect(distImportHelperIndex).toBeLessThan(installIndex);
expect(packageManifestIndex).toBeLessThan(installIndex);
expect(extensionManifestIndex).toBeLessThan(installIndex);
expect(exampleManifestIndex).toBeLessThan(installIndex);
});
it.runIf(process.platform !== "win32").each(["", "matrix"])(
"stages every locked workspace manifest while retaining optional plugin selection: %j",
async (selection) => {
const dockerfile = collapseDockerContinuations(await readFile(dockerfilePath, "utf8"));
const stages = new Map(
[...dockerfile.matchAll(/^FROM \S+ AS (\S+)\n([\s\S]*?)(?=^FROM |(?![\s\S]))/gm)].map(
([, name, body]) => [name, body],
),
);
const workspaceBody = stages.get("workspace-deps");
const dependencyBody = stages.get("dependency-inputs");
if (!workspaceBody || !dependencyBody) {
throw new Error("Docker dependency input stages are missing");
}
const fixture = tempDirs.make("openclaw-docker-workspace-manifests-");
const workspaceRoot = await stageDockerManifestCopies({
body: workspaceBody,
fixture,
stage: "workspace-deps",
});
const extraction = [...workspaceBody.matchAll(/^RUN (.+)$/gm)]
.map(([, command]) => command)
.join("\n")
.replace(/\/(tmp|out)(?=\/|["\s]|$)/g, `${workspaceRoot}/$1`);
expect(extraction).not.toBe("");
execFileSync("/bin/sh", ["-eu", "-c", extraction], {
env: {
PATH: process.env.PATH,
OPENCLAW_EXTENSIONS: selection,
OPENCLAW_BUNDLED_PLUGIN_DIR: BUNDLED_PLUGIN_ROOT_DIR,
},
});
const inputsRoot = await stageDockerManifestCopies({
body: dependencyBody,
fixture,
stage: "dependency-inputs",
});
const appRoot = join(inputsRoot, "app");
const documents = parseAllDocuments(await readFile(join(appRoot, "pnpm-lock.yaml"), "utf8"));
expect(documents.flatMap((document) => document.errors)).toEqual([]);
const importers = documents.at(-1)?.get("importers", true);
if (!isMap(importers)) {
throw new Error("Application pnpm lockfile importers are missing");
}
expect(importers.items.length).toBeGreaterThan(1);
for (const importer of importers.items) {
const manifest = join(String(importer.key), "package.json");
expect(
await readFile(join(appRoot, manifest)).catch(() => null),
`Docker dependency inputs are missing ${manifest}`,
).toEqual(await readFile(join(repoRoot, manifest)));
}
const selected = await readFile(
join(inputsRoot, "tmp/openclaw-selected-plugin-dirs"),
"utf8",
);
expect(selected).toBe(selection ? `${selection}\n` : "");
const installed = (
await readFile(join(inputsRoot, "tmp/openclaw-workspace-plugin-dirs"), "utf8")
)
.trim()
.split("\n");
expect(installed).toContain("telegram");
expect(installed).not.toContain("facetime");
expect(installed.includes("matrix")).toBe(selection === "matrix");
},
);
it("keeps validated plugin selection outside the build-context copy destination", async () => {
const dockerfile = await readFile(dockerfilePath, "utf8");
const selectionCopyIndex = dockerfile.indexOf(