vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-11-11 18:42:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

refactor(codemode): name the data boundary and prepare tools once (#48021)

Browse source
This commit is contained in:
Aiden Cline 2026-09-08 21:35:36 -05:00 • committed by GitHub
parent 74b0fa9d1f
commit 08e28fb915
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 263 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

36
packages/codemode/src/codemode.ts
View file

@ -1,5 +1,5 @@
import { Effect, Schema } from "effect"
import { executeWithLimits } from "./interpreter/execute.js"
import { executeProgram } from "./interpreter/execute.js"
import { type Services, type ToolDescription, ToolRuntime } from "./tool-runtime.js"
import type { Tools } from "./tools.js"
@ -30,26 +30,23 @@ export type ResolvedExecutionLimits = {
readonly maxOutputBytes: number | undefined
}
/** Options for one CodeMode execution. */
export type ExecuteOptions<Provided extends Record<string, unknown> = {}> = {
/** Source for one program in the supported JavaScript subset. */
code: string
/** Configuration shared by `CodeMode.make` and `CodeMode.execute`. */
export type Options<Provided extends Record<string, unknown> = {}> = ToolRuntime.ToolCallHooks<Services<Provided>> & {
/** Explicit tools exposed to the program as `tools`. */
tools?: Provided & Tools<Services<Provided>>
/** Per-execution overrides for the default resource limits. */
/** Resource limits enforced on each execution. */
limits?: ExecutionLimits
/** Observes decoded tool input immediately before tool execution. */
onToolCallStart?: (call: ToolRuntime.ToolCallStarted) => Effect.Effect<void, never, Services<Provided>>
/** Observes each admitted tool call as it succeeds, fails, or is interrupted. */
onToolCallEnd?: (call: ToolRuntime.ToolCallEnded) => Effect.Effect<void, never, Services<Provided>>
}
/** Options for one CodeMode execution. */
export type ExecuteOptions<Provided extends Record<string, unknown> = {}> = Options<Provided> & {
/** Source for one program in the supported JavaScript subset. */
code: string
}
/** A JSON value that can cross the confined interpreter boundary. */
export type DataValue = Schema.Json
/** Configuration shared by `CodeMode.make` and `CodeMode.execute`. */
export type Options<Provided extends Record<string, unknown> = {}> = Omit<ExecuteOptions<Provided>, "code">
/** Schema for a host tool input containing CodeMode source. */
export const Input = Schema.Struct({ code: Schema.String })
export type Input = typeof Input.Type
@ -128,21 +125,16 @@ const resolveExecutionLimits = (limits?: ExecutionLimits): ResolvedExecutionLimi
/** Executes one Effect-native CodeMode program without constructing a reusable runtime. */
export const execute = <const Provided extends Record<string, unknown>>(
options: ExecuteOptions<Provided>,
): Effect.Effect<Result, never, Services<Provided>> => {
const tools = (options.tools ?? {}) as Tools<Services<Provided>>
return executeWithLimits(options, resolveExecutionLimits(options.limits), ToolRuntime.searchIndex(tools))
}
): Effect.Effect<Result, never, Services<Provided>> => make(options).execute(options.code)
/** Creates an Effect-native runtime over explicit, schema-described tools. */
export const make = <const Provided extends Record<string, unknown> = {}>(
options: Options<Provided> = {} as Options<Provided>,
options: Options<Provided> = {},
): Runtime<Services<Provided>> => {
const tools = (options.tools ?? {}) as Tools<Services<Provided>>
const prepared = ToolRuntime.prepare((options.tools ?? {}) as Tools<Services<Provided>>)
const limits = resolveExecutionLimits(options.limits)
const prepared = ToolRuntime.prepare(tools)
return {
catalog: () => prepared.catalog,
execute: (code) => executeWithLimits<Provided>({ ...options, code }, limits, prepared.searchIndex),
execute: (code) => executeProgram(code, prepared, limits, options),
}
}

147
packages/codemode/src/data.ts Normal file
View file

@ -0,0 +1,147 @@
export * as Data from "./data.js"
import type { DiagnosticKind } from "./codemode.js"
import { Values } from "./values.js"
/** A null-prototype object owned by the program. */
export type SafeObject = Record<string, unknown>
const MAX_VALUE_DEPTH = 32
export class ToolRuntimeError extends Error {
constructor(
readonly kind: Extract<
DiagnosticKind,
"UnknownTool" | "InvalidToolInput" | "InvalidToolOutput" | "InvalidDataValue" | "ToolCallLimitExceeded"
>,
message: string,
readonly suggestions: ReadonlyArray<string> = [],
) {
super(message)
this.name = "ToolRuntimeError"
}
}
const blockedMemberNames = new Set(["__proto__", "constructor", "prototype"])
export const isBlockedMember = (name: string): boolean => blockedMemberNames.has(name)
/**
* Brings a host-produced runtime value into the program: runtime values pass through, their host
* counterparts (Date, RegExp, Map, Set, URL, URLSearchParams) are wrapped, and objects become
* null-prototype copies. Arrays keep extra enumerable properties such as `index` and `groups`.
*/
export const toProgram = (value: unknown, label: string): unknown => copy(value, label, "program", 0, new Set())
/**
* Brings host data into the program: Date and URL become strings, other host collections become
* empty objects, and objects become null-prototype copies. Used for tool results and parsed JSON.
*/
export const fromData = (value: unknown, label: string): unknown => copy(value, label, "data", 0, new Set())
/**
* Takes a program value out as plain JSON: runtime values serialize like `JSON.stringify` would,
* non-finite numbers become null, and array holes become null. `undefined` object properties are
* dropped ("json") or become null ("result", for program results where the consumer must never see
* undefined); a bare `undefined` follows the same rule.
*/
export const toData = (value: unknown, label: string, undefinedAs: "json" | "result" = "json"): unknown =>
copy(value, label, undefinedAs, 0, new Set())
// "program" and "data" build program-owned null-prototype objects; "json" and "result" build
// ordinary objects for the host.
type Mode = "program" | "data" | "json" | "result"
const copy = (value: unknown, label: string, mode: Mode, depth: number, seen: Set<object>): unknown => {
if (depth > MAX_VALUE_DEPTH) {
throw new ToolRuntimeError("InvalidDataValue", `${label} exceeds the maximum value depth of ${MAX_VALUE_DEPTH}.`)
}
if (value === undefined) return mode === "result" ? null : undefined
if (typeof value === "number") return (mode === "json" || mode === "result") && !Number.isFinite(value) ? null : value
if (value === null || typeof value === "string" || typeof value === "boolean") return value
if (typeof value !== "object") {
throw new ToolRuntimeError("InvalidDataValue", `${label} must contain data only.`)
}
if (value instanceof Values.Promise) {
throw new ToolRuntimeError(
"InvalidDataValue",
`${label} contains an un-awaited Promise; await tool calls (e.g. \`const result = await tools.ns.tool(...)\`) before using their results.`,
)
}
if (mode === "program") {
if (Values.isValue(value)) return value
if (value instanceof Date) return new Values.Date(value.getTime())
if (value instanceof RegExp) return new Values.RegExp(value.source, value.flags)
if (value instanceof Map) {
const wrapped = new Values.Map()
for (const [key, item] of value.entries()) {
wrapped.map.set(copy(key, label, mode, depth + 1, seen), copy(item, label, mode, depth + 1, seen))
}
return wrapped
}
if (value instanceof Set) {
const wrapped = new Values.Set()
for (const item of value.values()) wrapped.set.add(copy(item, label, mode, depth + 1, seen))
return wrapped
}
if (value instanceof URL) return new Values.URL(new URL(value.href))
if (value instanceof URLSearchParams) return new Values.URLSearchParams(new URLSearchParams(value))
}
const plain = mode === "program" || mode === "data"
if (value instanceof Values.Date) return Number.isFinite(value.time) ? new Date(value.time).toISOString() : null
if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.toISOString() : null
if (value instanceof Values.URL) return value.url.href
if (value instanceof URL) return value.href
// Remaining runtime values and their host counterparts serialize as empty objects, like JSON.stringify.
if (
Values.isValue(value) ||
value instanceof RegExp ||
value instanceof Map ||
value instanceof Set ||
value instanceof URLSearchParams
) {
return plain ? (Object.create(null) as SafeObject) : {}
}
if (seen.has(value)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains a circular value.`)
}
seen.add(value)
if (Array.isArray(value)) {
// Host output densifies holes to null like JSON; program copies keep them.
const copied = plain
? value.map((item) => copy(item, label, mode, depth + 1, seen))
: Array.from(value, (item) => copy(item, label, mode, depth + 1, seen) ?? null)
if (mode === "program") {
for (const [key, item] of Object.entries(value)) {
if (Object.hasOwn(copied, key)) continue
if (isBlockedMember(key)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains blocked property '${key}'.`)
}
Reflect.set(copied, key, copy(item, label, mode, depth + 1, seen))
}
}
seen.delete(value)
return copied
}
const prototype = Object.getPrototypeOf(value)
if (prototype !== Object.prototype && prototype !== null) {
throw new ToolRuntimeError("InvalidDataValue", `${label} must contain plain objects only.`)
}
const copied: SafeObject = plain ? (Object.create(null) as SafeObject) : {}
for (const [key, item] of Object.entries(value)) {
if (isBlockedMember(key)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains blocked property '${key}'.`)
}
const next = copy(item, label, mode, depth + 1, seen)
if (next === undefined && mode === "json") continue
copied[key] = next
}
seen.delete(value)
return copied
}

4
packages/codemode/src/interpreter/errors.ts
View file

@ -1,7 +1,7 @@
import { Effect } from "effect"
import type { Diagnostic } from "../codemode.js"
import { ToolError } from "../tool-error.js"
import { copyOut, ToolRuntimeError, type SafeObject } from "../tool-runtime.js"
import { type SafeObject, toData, ToolRuntimeError } from "../data.js"
import { type AstNode, formatLocation, InterpreterRuntimeError, ProgramThrow, sourceLocation } from "./model.js"
import { containsRuntimeReference } from "./references.js"
import { type SyncIteratorRunner } from "./iterator.js"
@ -45,7 +45,7 @@ export const normalizeError = (error: unknown): Diagnostic => {
message = (value as { message: string }).message
} else {
try {
message = JSON.stringify(copyOut(value, "json")) ?? String(value)
message = JSON.stringify(toData(value, "Thrown value")) ?? String(value)
} catch {
message = String(value)
}

43
packages/codemode/src/interpreter/execute.ts
View file

@ -3,20 +3,21 @@ import { Cause, Effect, Scope } from "effect"
// #transpile: conditional import — full typescript on node/bun, an identity
// pass-through on workerd (the compiler is ~11 MiB and can't init there).
import { transpile } from "#transpile"
import type { DataValue, Diagnostic, ExecuteOptions, ResolvedExecutionLimits, Result } from "../codemode.js"
import { copyIn, copyOut, ToolRuntime, type Services } from "../tool-runtime.js"
import type { Tools } from "../tools.js"
import type { DataValue, Diagnostic, ResolvedExecutionLimits, Result } from "../codemode.js"
import { toData } from "../data.js"
import { ToolRuntime } from "../tool-runtime.js"
import { normalizeError } from "./errors.js"
import { InterpreterRuntimeError, isRecord, type ProgramNode } from "./model.js"
import { PromiseRuntime } from "./promises.js"
import { Interpreter } from "./runtime.js"
export const executeWithLimits = <const Provided extends Record<string, unknown>>(
options: ExecuteOptions<Provided>,
export const executeProgram = <R>(
code: string,
prepared: ToolRuntime.Prepared<R>,
limits: ResolvedExecutionLimits,
searchIndex: ToolRuntime.DiscoveryPlan["searchIndex"],
): Effect.Effect<Result, never, Services<Provided>> => {
if (options.code.trim().length === 0) {
hooks: ToolRuntime.ToolCallHooks<R>,
): Effect.Effect<Result, never, R> => {
if (code.trim().length === 0) {
return Effect.succeed({
ok: false,
error: { kind: "ParseError", message: "Code cannot be empty." },
@ -26,35 +27,21 @@ export const executeWithLimits = <const Provided extends Record<string, unknown>
// Allocate execution state inside suspension so reused Effects never share it.
return Effect.suspend(() => {
const tools = ToolRuntime.make(
(options.tools ?? {}) as Tools<Services<Provided>>,
limits.maxToolCalls,
searchIndex,
{
onToolCallStart: options.onToolCallStart,
onToolCallEnd: options.onToolCallEnd,
},
)
const tools = ToolRuntime.make(prepared, limits.maxToolCalls, hooks)
const logs: Array<string> = []
const logged = () => (logs.length > 0 ? { logs: [...logs] } : {})
// Set only after copy-out so timeouts cannot report invalid values as completed.
let returned: { value: DataValue; promises: PromiseRuntime<Services<Provided>> } | undefined
let returned: { value: DataValue; promises: PromiseRuntime<R> } | undefined
const base = Effect.acquireUseRelease(
Scope.make("parallel"),
(scope) =>
Effect.gen(function* () {
const program = parseProgram(options.code)
const promises = new PromiseRuntime<Services<Provided>>(scope)
const interpreter = new Interpreter<Services<Provided>>(
tools.execute,
tools.search,
tools.keys,
promises,
logs,
)
const program = parseProgram(code)
const promises = new PromiseRuntime<R>(scope)
const interpreter = new Interpreter<R>(tools.execute, tools.search, tools.keys, promises, logs)
const value = yield* interpreter.run(program)
const result = copyOut(copyIn(value, "Execution result"), "nullify") as DataValue
const result = toData(value, "Execution result", "result") as DataValue
returned = { value: result, promises }
const warnings = yield* promises.interrupt()
return {

17
packages/codemode/src/interpreter/methods.ts
View file

@ -15,7 +15,8 @@ import {
UriFunction,
} from "./model.js"
import { containsOpaqueReference, isRuntimeReference, rejectCircularInsertion, typeofValue } from "./references.js"
import { compareText, isBlockedMember, type SafeObject } from "../tool-runtime.js"
import { isBlockedMember, type SafeObject, toProgram } from "../data.js"
import { compareText } from "../tool-runtime.js"
import { Values } from "../values.js"
import { dateSetterArgumentCount, invokeDateMethod, invokeDateStatic } from "../stdlib/date.js"
import { invokeMathMethod } from "../stdlib/math.js"
@ -24,7 +25,7 @@ import { invokeObjectMethod } from "../stdlib/object.js"
import { invokeRegExpMethod, invokeRegExpStatic, matchToValue, toHostRegex } from "../stdlib/regexp.js"
import { invokeStringStatic } from "../stdlib/string.js"
import { invokeURLMethod, invokeURLStatic, uriArgument } from "../stdlib/url.js"
import { boundedData, coerceToNumber, coerceToString, errorBrandName } from "../stdlib/value.js"
import { coerceToNumber, coerceToString, errorBrandName } from "../stdlib/value.js"
import { preserveConsumerError, type SyncIteratorRunner } from "./iterator.js"
export type CallbackRunner<R> = {
@ -292,7 +293,7 @@ const invokeStringMethod = (value: string, name: string, args: Array<unknown>, n
const matched = value.match(pattern)
if (matched === null) return null
// Preserve the own `index` and `groups` properties on non-global matches.
if (pattern.global) return boundedData(matched, "String.match result")
if (pattern.global) return toProgram(matched, "String.match result")
return matchToValue(matched)
}
case "matchAll": {
@ -347,7 +348,7 @@ const invokeStringMethod = (value: string, name: string, args: Array<unknown>, n
default:
throw new InterpreterRuntimeError(`String method '${name}' is not available.`, node)
}
return boundedData(result, `String.${name} result`)
return toProgram(result, `String.${name} result`)
}
export const arrayStatics = new Set(["isArray", "of", "from"])
@ -540,19 +541,19 @@ const invokeStringReplacer = <R>(
let end = 0
for (const match of matches) {
const replacement = yield* apply(match.args)
// Error values are branded plain objects; boundedData would strip the brand before coercion.
// Error values are branded plain objects; toProgram would strip the brand before coercion.
output.push(
value.slice(end, match.offset),
replacement instanceof Values.Promise
? "[object Promise]"
: errorBrandName(replacement)
? coerceToString(replacement)
: coerceToString(boundedData(replacement, `String.${name} replacer result`)),
: coerceToString(toProgram(replacement, `String.${name} replacer result`)),
)
end = match.offset + match.match.length
}
output.push(value.slice(end))
return boundedData(output.join(""), `String.${name} result`)
return toProgram(output.join(""), `String.${name} result`)
})
}
@ -876,7 +877,7 @@ const invokeArrayMethod = <R>(
if (args.length > 1 || (args.length === 1 && typeof args[0] !== "string")) {
throw new InterpreterRuntimeError("Array.join expects zero arguments or one string separator.", node)
}
const input = boundedData(target, "Array.join input") as Array<unknown>
const input = toProgram(target, "Array.join input") as Array<unknown>
return Effect.succeed(
input.map((item) => coerceToString(item ?? "")).join(args.length === 0 ? "," : (args[0] as string)),
)

2
packages/codemode/src/interpreter/model.ts
View file

@ -1,6 +1,6 @@
import type { Effect } from "effect"
import type { DiagnosticKind } from "../codemode.js"
import type { SafeObject } from "../tool-runtime.js"
import type { SafeObject } from "../data.js"
import type { Values } from "../values.js"
export type SourcePosition = {

2
packages/codemode/src/interpreter/promises.ts
View file

@ -1,6 +1,6 @@
import { Cause, Deferred, Effect, Exit, Fiber, Scope } from "effect"
import type { Diagnostic } from "../codemode.js"
import type { SafeObject } from "../tool-runtime.js"
import type { SafeObject } from "../data.js"
import {
type AstNode,
CodeModeFunction,

24
packages/codemode/src/interpreter/runtime.ts
View file

@ -1,5 +1,6 @@
import { Cause, Deferred, Effect, Exit } from "effect"
import { isBlockedMember, ToolReference, ToolRuntimeError, type SafeObject } from "../tool-runtime.js"
import { isBlockedMember, ToolRuntimeError, type SafeObject, toProgram } from "../data.js"
import { ToolReference } from "../tool-runtime.js"
import {
type AstNode,
AsyncIteratorSymbol,
@ -90,7 +91,6 @@ import {
urlArgument,
} from "../stdlib/url.js"
import {
boundedData,
coerceToNumber,
coerceToString,
compoundOperators,
@ -1342,7 +1342,7 @@ export class Interpreter<R> {
this.constructRegExp([regex.pattern, typeof regex.flags === "string" ? regex.flags : ""], node),
)
}
return Effect.sync(() => boundedData(node.value, "Literal"))
return Effect.sync(() => toProgram(node.value, "Literal"))
}
case "Identifier":
return Effect.sync(() => this.scopes.get(getString(node, "name"), node))
@ -1614,7 +1614,7 @@ export class Interpreter<R> {
).as("TypeError")
}
if (Values.isValue(init)) return new Values.URLSearchParams(new URLSearchParams())
const data = boundedData(init, "new URLSearchParams input")
const data = toProgram(init, "new URLSearchParams input")
if (data === null || typeof data !== "object") {
throw new InterpreterRuntimeError(
"new URLSearchParams(...) expects a query string, data object, iterable pairs, or URLSearchParams.",
@ -1659,7 +1659,7 @@ export class Interpreter<R> {
const lhs = yield* self.evaluateExpression(getNode(node, "left"))
const rhs = yield* self.evaluateExpression(getNode(node, "right"))
if (operator === "instanceof") return instanceofValue(lhs, rhs, node)
return boundedData(self.applyBinaryOperator(operator, lhs, rhs, node), "Binary expression result")
return toProgram(self.applyBinaryOperator(operator, lhs, rhs, node), "Binary expression result")
})
}
@ -1776,7 +1776,7 @@ export class Interpreter<R> {
default:
throw new InterpreterRuntimeError(`Unsupported unary operator '${operator}'.`, node)
}
return boundedData(result, "Unary expression result")
return toProgram(result, "Unary expression result")
})
}
@ -1798,7 +1798,7 @@ export class Interpreter<R> {
if (operator !== "=") {
const current = self.scopes.get(name, left)
const rightValue = yield* self.evaluateExpression(getNode(node, "right"))
const next = boundedData(
const next = toProgram(
self.applyCompoundAssignment(operator, current, rightValue, node),
"Assignment result",
)
@ -1811,7 +1811,7 @@ export class Interpreter<R> {
return yield* self.modifyMember(left, (current) =>
Effect.map(self.evaluateExpression(getNode(node, "right")), (rightValue) => {
if (operator === "=") return { write: true, next: rightValue, result: rightValue }
const next = boundedData(
const next = toProgram(
self.applyCompoundAssignment(operator, current, rightValue, node),
"Assignment result",
)
@ -1961,13 +1961,13 @@ export class Interpreter<R> {
if (callable.namespace === "Array" && callable.name === "of") {
return invokeGlobalMethod(callable, args, node)
}
return boundedData(invokeGlobalMethod(callable, args, node), `${callable.namespace}.${callable.name} result`)
return toProgram(invokeGlobalMethod(callable, args, node), `${callable.namespace}.${callable.name} result`)
}
if (callable instanceof JsonMethodReference) {
return yield* invokeJsonMethod(self.runner, callable.name, args, node)
}
if (callable instanceof CoercionFunction) {
return boundedData(invokeCoercion(callable, args, node), `${callable.name} result`)
return toProgram(invokeCoercion(callable, args, node), `${callable.name} result`)
}
if (callable instanceof UriFunction) {
return invokeUriFunction(callable, args, node)
@ -2014,7 +2014,7 @@ export class Interpreter<R> {
private invokeObjectMethodOnTools(name: string, ref: ToolReference, node: AstNode): unknown {
if (name === "keys") {
return boundedData(this.enumerableKeys(ref)!, "Object.keys result")
return toProgram(this.enumerableKeys(ref)!, "Object.keys result")
}
throw new InterpreterRuntimeError(
`Object.${name}(...) cannot read tool references: they are not plain data. Use Object.keys(tools) for names, or search({ query }) for signatures.`,
@ -2445,7 +2445,7 @@ export class Interpreter<R> {
if (index < expressions.length) {
const raw = yield* self.evaluateExpression(asNode(expressions[index], "expressions"))
output += coerceToString(boundedData(raw, "Template interpolation"))
output += coerceToString(toProgram(raw, "Template interpolation"))
}
}

2
packages/codemode/src/openapi/spec.ts
View file

@ -1,6 +1,6 @@
import { fromSchemaOpenApi3_0, fromSchemaOpenApi3_1 } from "effect/JsonSchema"
import type { JsonSchema } from "../tool.js"
import { isBlockedMember } from "../tool-runtime.js"
import { isBlockedMember } from "../data.js"
import type {
Body,
Document,

8
packages/codemode/src/stdlib/console.ts
View file

@ -1,7 +1,7 @@
import { toData, toProgram } from "../data.js"
import { containsOpaqueReference, containsRuntimeReference, isRuntimeReference } from "../interpreter/references.js"
import { copyIn, copyOut } from "../tool-runtime.js"
import { Values } from "../values.js"
import { boundedData, coerceToString } from "./value.js"
import { coerceToString } from "./value.js"
export const consoleMethods = new Set(["log", "info", "debug", "warn", "error", "dir", "table"])
@ -66,7 +66,7 @@ const formatConsoleValue = (value: unknown, seen: Set<object>, depth: number): s
const formatConsoleTable = (value: unknown, columnsArgument: unknown): string => {
if (value === undefined) return "undefined"
if (containsOpaqueReference(value)) return "[opaque reference]"
const data = boundedData(value, "console.table argument")
const data = toProgram(value, "console.table argument")
const columns = consoleTableColumns(columnsArgument)
const rows = consoleTableRows(data, columns)
const keys = columns ?? Array.from(new Set(rows.flatMap((row) => Object.keys(row.values))))
@ -80,7 +80,7 @@ const formatConsoleTable = (value: unknown, columnsArgument: unknown): string =>
const consoleTableColumns = (value: unknown): ReadonlyArray<string> | undefined => {
if (value === undefined) return undefined
if (containsRuntimeReference(value)) return undefined
const columns = copyOut(copyIn(value, "console.table columns"), "nullify")
const columns = toData(value, "console.table columns", "result")
return Array.isArray(columns) ? columns.map((column) => String(column)) : undefined
}

28
packages/codemode/src/stdlib/json.ts
View file

@ -3,7 +3,7 @@ import type { CallbackRunner } from "../interpreter/methods.js"
import { applyCollectionCallback } from "../interpreter/methods.js"
import { type AstNode, InterpreterRuntimeError } from "../interpreter/model.js"
import { typeofValue } from "../interpreter/references.js"
import { copyIn, copyOut, type SafeObject } from "../tool-runtime.js"
import { fromData, type SafeObject, toData, toProgram } from "../data.js"
import { Values } from "../values.js"
export const jsonStatics = new Set(["parse", "stringify"])
@ -28,7 +28,7 @@ const parse = <R>(
const parsed = (() => {
try {
return copyIn(JSON.parse(text), "JSON.parse result")
return fromData(JSON.parse(text), "JSON.parse result")
} catch (error) {
throw new InterpreterRuntimeError(
`JSON.parse received invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
@ -71,29 +71,27 @@ const stringify = <R>(
const space = args[2]
const indent = typeof space === "number" || typeof space === "string" ? space : undefined
const replacer = args[1]
const callable = typeofValue(replacer) === "function"
const checked = copyIn(args[0], "JSON.stringify value", callable)
const input = callable ? args[0] : checked
if (Array.isArray(replacer)) {
const properties = replacer
.filter((item): item is string | number => typeof item === "string" || typeof item === "number")
.map(String)
return Effect.succeed(JSON.stringify(copyOut(input, "json"), properties, indent))
}
if (!callable) {
return Effect.succeed(JSON.stringify(copyOut(input, "json"), null, indent))
if (typeofValue(replacer) !== "function") {
const properties = Array.isArray(replacer)
? replacer
.filter((item): item is string | number => typeof item === "string" || typeof item === "number")
.map(String)
: null
return Effect.succeed(JSON.stringify(toData(args[0], "JSON.stringify value"), properties, indent))
}
// Validate up front; the replacer walk below reads the original value.
toProgram(args[0], "JSON.stringify value")
const apply = applyCollectionCallback(runner, replacer, "JSON.stringify", node)
const root: SafeObject = Object.create(null) as SafeObject
root[""] = input
root[""] = args[0]
const stack = new Set<object>()
const visit = (holder: SafeObject | Array<unknown>, key: string): Effect.Effect<unknown, unknown, R> =>
Effect.gen(function* () {
const value = yield* apply([key, toJSONValue(holder[key as keyof typeof holder])])
if (value === undefined || typeofValue(value) === "function") return undefined
copyIn(value, "JSON.stringify replacer result", true)
toProgram(value, "JSON.stringify replacer result")
if (typeof value === "number") return Number.isFinite(value) ? value : null
if (value === null || typeof value === "string" || typeof value === "boolean") return value
if (Array.isArray(value)) {

5
packages/codemode/src/stdlib/number.ts
View file

@ -1,5 +1,6 @@
import { type AstNode, InterpreterRuntimeError } from "../interpreter/model.js"
import { boundedData, coerceToString } from "./value.js"
import { toProgram } from "../data.js"
import { coerceToString } from "./value.js"
export const numberMethods = new Set(["toFixed", "toPrecision", "toExponential", "toString", "valueOf"])
@ -50,7 +51,7 @@ export const invokeNumberMethod = (value: number, name: string, args: Array<unkn
default:
throw new InterpreterRuntimeError(`Number method '${name}' is not available.`, node)
}
return boundedData(result, `Number.${name} result`)
return toProgram(result, `Number.${name} result`)
}
export const invokeNumberStatic = (name: string, args: Array<unknown>, node: AstNode): unknown => {

9
packages/codemode/src/stdlib/object.ts
View file

@ -1,9 +1,10 @@
import { Effect } from "effect"
import { type AstNode, AsyncIteratorSymbol, InterpreterRuntimeError, IteratorSymbol } from "../interpreter/model.js"
import { containsOpaqueReference, rejectCircularInsertion } from "../interpreter/references.js"
import { isBlockedMember } from "../tool-runtime.js"
import { isBlockedMember } from "../data.js"
import { Values } from "../values.js"
import { boundedData, coerceToString } from "./value.js"
import { toProgram } from "../data.js"
import { coerceToString } from "./value.js"
import { preserveConsumerError, type SyncIteratorRunner } from "../interpreter/iterator.js"
export const objectMethodsPreservingIdentity = new Set(["assign", "values", "entries", "fromEntries"])
@ -115,8 +116,8 @@ export const invokeObjectFromEntries = <R>(
)
}
const entry = step.value as Record<string, unknown>
boundedData(entry[0], "Object.fromEntries key")
boundedData(entry[1], "Object.fromEntries value")
toProgram(entry[0], "Object.fromEntries key")
toProgram(entry[1], "Object.fromEntries value")
const key = coerceToString(entry[0])
if (isBlockedMember(key)) throw new InterpreterRuntimeError(`Property '${key}' is not available.`, node)
out[key] = entry[1]

2
packages/codemode/src/stdlib/regexp.ts
View file

@ -1,5 +1,5 @@
import { type AstNode, InterpreterRuntimeError } from "../interpreter/model.js"
import { isBlockedMember, type SafeObject } from "../tool-runtime.js"
import { isBlockedMember, type SafeObject } from "../data.js"
import { Values } from "../values.js"
import { coerceToNumber, coerceToString } from "./value.js"

5
packages/codemode/src/stdlib/url.ts
View file

@ -1,6 +1,7 @@
import { type AstNode, InterpreterRuntimeError, UriFunction } from "../interpreter/model.js"
import { Values } from "../values.js"
import { boundedData, coerceToString } from "./value.js"
import { toProgram } from "../data.js"
import { coerceToString } from "./value.js"
export const urlProperties = new Set([
"href",
@ -46,7 +47,7 @@ export const urlSearchParamsMethods = new Set([
"toString",
])
export const uriArgument = (value: unknown, label: string): string => coerceToString(boundedData(value, label))
export const uriArgument = (value: unknown, label: string): string => coerceToString(toProgram(value, label))
export const invokeUriFunction = (ref: UriFunction, args: Array<unknown>, node: AstNode): string => {
const value = uriArgument(args[0], `${ref.name} input`)

8
packages/codemode/src/stdlib/value.ts
View file

@ -1,5 +1,5 @@
import { type AstNode, CoercionFunction, InterpreterRuntimeError } from "../interpreter/model.js"
import { copyIn, type SafeObject } from "../tool-runtime.js"
import { type SafeObject, toProgram } from "../data.js"
import { Values } from "../values.js"
export const errorConstructors = new Set([
@ -33,8 +33,6 @@ export const errorBrandName = (value: unknown): string | undefined =>
? ((value as Record<PropertyKey, unknown>)[ErrorBrand] as string | undefined)
: undefined
export const boundedData = (value: unknown, label: string): unknown => copyIn(value, label, true)
export const coerceToString = (value: unknown): string => {
if (value === null) return "null"
if (value === undefined) return "undefined"
@ -79,7 +77,7 @@ export const invokeCoercion = (ref: CoercionFunction, args: Array<unknown>, node
if (ref.name === "String") return ""
}
const raw = args[0]
// Error values are plain SafeObjects; the boundedData path below would strip their brand.
// Error values are plain SafeObjects; the toProgram path below would strip their brand.
if (ref.name === "String" && errorBrandName(raw) !== undefined) return coerceToString(raw)
if (Values.isValue(raw)) {
if (ref.name === "Boolean") return true
@ -90,7 +88,7 @@ export const invokeCoercion = (ref: CoercionFunction, args: Array<unknown>, node
if (ref.name === "parseInt") return parseInt(coerceToString(raw))
return parseFloat(coerceToString(raw))
}
const value = boundedData(raw, `${ref.name} input`)
const value = toProgram(raw, `${ref.name} input`)
if (ref.name === "Number") return coerceToNumber(value)
if (ref.name === "Boolean") return Boolean(value)
if (ref.name === "isFinite") return Number.isFinite(coerceToNumber(value))

206
packages/codemode/src/tool-runtime.ts
View file

@ -1,5 +1,5 @@
import { Cause, Effect, Exit, Formatter, Schema } from "effect"
import type { DiagnosticKind } from "./codemode.js"
import { fromData, toData, ToolRuntimeError } from "./data.js"
import { toolError } from "./tool-error.js"
import {
decodeInput as decodeToolInput,
@ -13,7 +13,6 @@ import {
import { isNamespace, type Namespace } from "./namespace.js"
import { isTool, type Tool } from "./tool.js"
import type { Tools } from "./tools.js"
import { Values } from "./values.js"
export const compareText = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0)
@ -52,7 +51,9 @@ export type ToolCallEnded = {
}
export type ToolCallHooks<R = never> = {
/** Observes decoded tool input immediately before tool execution. */
readonly onToolCallStart?: ((call: ToolCallStarted) => Effect.Effect<void, never, R>) | undefined
/** Observes each admitted tool call as it succeeds, fails, or is interrupted. */
readonly onToolCallEnd?: ((call: ToolCallEnded) => Effect.Effect<void, never, R>) | undefined
}
@ -62,8 +63,6 @@ export type ToolDescription = {
readonly signature: string
}
export type SafeObject = Record<string, unknown>
const defaultSearchLimit = 10
const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0))
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))
@ -94,168 +93,6 @@ export class ToolReference {
constructor(readonly path: ReadonlyArray<string>) {}
}
const MAX_VALUE_DEPTH = 32
export class ToolRuntimeError extends Error {
constructor(
readonly kind: Extract<
DiagnosticKind,
"UnknownTool" | "InvalidToolInput" | "InvalidToolOutput" | "InvalidDataValue" | "ToolCallLimitExceeded"
>,
message: string,
readonly suggestions: ReadonlyArray<string> = [],
) {
super(message)
this.name = "ToolRuntimeError"
}
}
const blockedMemberNames = new Set(["__proto__", "constructor", "prototype"])
export const isBlockedMember = (name: string): boolean => blockedMemberNames.has(name)
// Checkpoint mode preserves CodeMode values; boundary mode JSON-normalizes them.
export const copyIn = (value: unknown, label: string, preserveCodeModeValues = false): unknown =>
copyBounded(value, label, 0, new Set(), preserveCodeModeValues)
const copyBounded = (
value: unknown,
label: string,
depth: number,
seen: Set<object>,
preserveCodeModeValues: boolean,
): unknown => {
if (depth > MAX_VALUE_DEPTH) {
throw new ToolRuntimeError("InvalidDataValue", `${label} exceeds the maximum value depth of ${MAX_VALUE_DEPTH}.`)
}
if (
value === null ||
value === undefined ||
typeof value === "string" ||
typeof value === "boolean" ||
typeof value === "number"
) {
return value
}
if (typeof value !== "object") {
throw new ToolRuntimeError("InvalidDataValue", `${label} must contain data only.`)
}
if (value instanceof Values.Promise) {
throw new ToolRuntimeError(
"InvalidDataValue",
`${label} contains an un-awaited Promise; await tool calls (e.g. \`const result = await tools.ns.tool(...)\`) before using their results.`,
)
}
if (preserveCodeModeValues) {
if (Values.isValue(value)) return value
if (value instanceof Date) return new Values.Date(value.getTime())
if (value instanceof RegExp) return new Values.RegExp(value.source, value.flags)
if (value instanceof Map) {
const wrapped = new Values.Map()
for (const [key, item] of value.entries()) {
wrapped.map.set(copyBounded(key, label, depth + 1, seen, true), copyBounded(item, label, depth + 1, seen, true))
}
return wrapped
}
if (value instanceof Set) {
const wrapped = new Values.Set()
for (const item of value.values()) wrapped.set.add(copyBounded(item, label, depth + 1, seen, true))
return wrapped
}
if (value instanceof URL) return new Values.URL(new URL(value.href))
if (value instanceof URLSearchParams) return new Values.URLSearchParams(new URLSearchParams(value))
}
if (value instanceof Values.Date) {
return Number.isFinite(value.time) ? new Date(value.time).toISOString() : null
}
if (value instanceof Date) {
return Number.isFinite(value.getTime()) ? value.toISOString() : null
}
if (value instanceof Values.URL) return value.url.href
if (value instanceof URL) return value.href
// Remaining runtime values and their host counterparts serialize as empty objects, like JSON.stringify.
if (
Values.isValue(value) ||
value instanceof RegExp ||
value instanceof Map ||
value instanceof Set ||
value instanceof URLSearchParams
) {
return Object.create(null) as SafeObject
}
if (seen.has(value)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains a circular value.`)
}
seen.add(value)
if (Array.isArray(value)) {
const copied = value.map((item) => copyBounded(item, label, depth + 1, seen, preserveCodeModeValues))
if (preserveCodeModeValues) {
// Checkpoint copies retain array metadata that boundary copies omit.
for (const [key, item] of Object.entries(value)) {
if (Object.hasOwn(copied, key)) continue
if (isBlockedMember(key)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains blocked property '${key}'.`)
}
Reflect.set(copied, key, copyBounded(item, label, depth + 1, seen, true))
}
}
seen.delete(value)
return copied
}
const prototype = Object.getPrototypeOf(value)
if (prototype !== Object.prototype && prototype !== null) {
throw new ToolRuntimeError("InvalidDataValue", `${label} must contain plain objects only.`)
}
const copied: SafeObject = Object.create(null) as SafeObject
for (const [key, item] of Object.entries(value)) {
if (isBlockedMember(key)) {
throw new ToolRuntimeError("InvalidDataValue", `${label} contains blocked property '${key}'.`)
}
copied[key] = copyBounded(item, label, depth + 1, seen, preserveCodeModeValues)
}
seen.delete(value)
return copied
}
// "json" mirrors JSON.stringify (undefined object values drop, undefined array elements become
// null, a bare undefined passes through): use it wherever data leaves as JSON, like tool
// arguments and stringify-style formatting. "nullify" turns every undefined, including a bare
// one, into null: use it for program results, where the consumer must never see undefined.
export type CopyOutMode = "json" | "nullify"
export const copyOut = (value: unknown, mode: CopyOutMode): unknown => {
if (value === undefined && mode === "nullify") return null
if (typeof value === "number" && !Number.isFinite(value)) {
return null
}
if (Array.isArray(value)) {
// Array.from densifies holes so sparse arrays normalize at the boundary like JSON does.
return Array.from(value, (item) => {
const copied = copyOut(item, mode)
return copied === undefined && mode === "json" ? null : copied
})
}
if (value !== null && typeof value === "object" && !(value instanceof ToolReference)) {
return Object.fromEntries(
Object.entries(value)
.map(([key, item]) => [key, copyOut(item, mode)] as const)
.filter(([, item]) => !(item === undefined && mode === "json")),
)
}
return value
}
// Dots in tool names are namespace separators; the last tool for a canonical path wins.
type ToolNode<R> = {
tool?: Tool<R>
@ -314,11 +151,9 @@ const describeTool = <R>(visible: VisibleTool<R>): ToolDescription => ({
: `${toolExpression(visible.path)}(input: ${inputTypeScript(visible.tool, true)}): Promise<${outputTypeScript(visible.tool, true)}>`,
})
// Discovery bytes are durable instructions, so order only after canonical-path collisions settle.
const visibleTools = <R>(tools: Tools<R>) =>
flattenTools(toolTrie(tools)).sort((left, right) => compareText(left.path, right.path))
export type DiscoveryPlan = {
/** Tools indexed once per runtime: the lookup trie plus the model-facing catalog and search index. */
export type Prepared<R = never> = {
readonly root: ToolNode<R>
readonly catalog: ReadonlyArray<ToolDescription>
readonly searchIndex: ReadonlyArray<SearchEntry>
}
@ -426,11 +261,12 @@ const toSearchEntry = <R>(visible: VisibleTool<R>): SearchEntry => ({
.toLowerCase(),
})
export const searchIndex = <R>(tools: Tools<R>): ReadonlyArray<SearchEntry> => visibleTools(tools).map(toSearchEntry)
export const prepare = <R>(tools: Tools<R>): DiscoveryPlan => {
const visible = visibleTools(tools)
export const prepare = <R>(tools: Tools<R>): Prepared<R> => {
const root = toolTrie(tools)
// Discovery bytes are durable instructions, so order only after canonical-path collisions settle.
const visible = flattenTools(root).sort((left, right) => compareText(left.path, right.path))
return {
root,
catalog: visible.map(describeTool),
searchIndex: visible.map(toSearchEntry),
}
@ -463,22 +299,21 @@ const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>): Tool<R> =>
}
export type ToolRuntime<R = never> = {
readonly root: ToolReference
readonly calls: Array<ToolCall>
readonly execute: (path: ReadonlyArray<string>, args: Array<unknown>) => Effect.Effect<unknown, unknown, R>
readonly search: (args: Array<unknown>) => Effect.Effect<unknown, unknown, R>
readonly keys: (path: ReadonlyArray<string>) => ReadonlyArray<string>
}
/** Per-execution call state over tools prepared once for the runtime. */
export const make = <R>(
tools: Tools<R>,
prepared: Prepared<R>,
maxToolCalls: number | undefined,
searchIndex: ReadonlyArray<SearchEntry>,
hooks?: ToolCallHooks<R>,
): ToolRuntime<R> => {
const calls: Array<ToolCall> = []
const root = toolTrie(tools)
const searchTool = makeSearchTool(searchIndex)
const root = prepared.root
const searchTool = makeSearchTool(prepared.searchIndex)
const observeEnd = <A, E>(effect: Effect.Effect<A, E, R>, call: ToolCallStarted): Effect.Effect<A, E, R> => {
const onEnd = hooks?.onToolCallEnd
@ -538,7 +373,7 @@ export const make = <R>(
}),
)
return yield* Effect.try({
try: () => copyIn(decodeToolOutput(tool, raw), `Result from tool '${name}'`),
try: () => fromData(decodeToolOutput(tool, raw), `Result from tool '${name}'`),
catch: (cause) => new ToolRuntimeError("InvalidToolOutput", `Invalid output from tool '${name}': ${cause}`),
})
}),
@ -547,21 +382,16 @@ export const make = <R>(
})
return {
root: new ToolReference([]),
calls,
keys: (path) => namespaceKeys(root, path),
search: (args) =>
Effect.suspend(() =>
executeTool(
"search",
searchTool,
args.map((arg) => copyOut(copyIn(arg, "Arguments for tool 'search'"), "json")),
),
executeTool("search", searchTool, args.map((arg) => toData(arg, "Arguments for tool 'search'"))),
),
execute: (path, args) =>
Effect.gen(function* () {
const name = canonicalSegments(path).join(".")
const externalArgs = args.map((arg) => copyOut(copyIn(arg, `Arguments for tool '${name}'`), "json"))
const externalArgs = args.map((arg) => toData(arg, `Arguments for tool '${name}'`))
const tool = resolve(root, path)
return yield* executeTool(name, tool, externalArgs)
}),

22
packages/codemode/test/parity.test.ts
View file

@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { CodeMode } from "../src/index.js"
import { ToolRuntime } from "../src/tool-runtime.js"
import { Data } from "../src/data.js"
// Runs a CodeMode program with no host tools and returns the CodeMode.Result. These tests pin the
// JS-parity behaviors for the "99% of ordinary defensive JavaScript just works" goal: cases where
@ -299,23 +299,23 @@ describe("H1: NaN/Infinity flow as intermediates and normalize to null at the bo
test("copyOut normalizes non-finite numbers to null (the shared return + tool-arg boundary)", () => {
// Tool-call arguments funnel through copyOut too, so this one function pins both boundaries.
expect(ToolRuntime.copyOut(NaN, "json")).toBeNull()
expect(ToolRuntime.copyOut(Infinity, "json")).toBeNull()
expect(ToolRuntime.copyOut(-Infinity, "nullify")).toBeNull()
expect(ToolRuntime.copyOut(42, "json")).toBe(42)
expect(ToolRuntime.copyOut({ a: NaN, b: [Infinity, 1] }, "json")).toEqual({ a: null, b: [null, 1] })
expect(Data.toData(NaN, "value")).toBeNull()
expect(Data.toData(Infinity, "value")).toBeNull()
expect(Data.toData(-Infinity, "value", "result")).toBeNull()
expect(Data.toData(42, "value")).toBe(42)
expect(Data.toData({ a: NaN, b: [Infinity, 1] }, "value")).toEqual({ a: null, b: [null, 1] })
})
})
describe("copyOut undefined handling per boundary mode", () => {
test("json mode mirrors JSON.stringify for undefined", () => {
expect(ToolRuntime.copyOut({ q: undefined, keep: 1 }, "json")).toStrictEqual({ keep: 1 })
expect(ToolRuntime.copyOut([1, undefined, 2], "json")).toStrictEqual([1, null, 2])
expect(ToolRuntime.copyOut({ nested: { a: undefined, b: [undefined] } }, "json")).toStrictEqual({
expect(Data.toData({ q: undefined, keep: 1 }, "value")).toStrictEqual({ keep: 1 })
expect(Data.toData([1, undefined, 2], "value")).toStrictEqual([1, null, 2])
expect(Data.toData({ nested: { a: undefined, b: [undefined] } }, "value")).toStrictEqual({
nested: { b: [null] },
})
expect(ToolRuntime.copyOut(undefined, "json")).toBeUndefined()
expect(ToolRuntime.copyOut({ a: undefined }, "nullify")).toStrictEqual({ a: null })
expect(Data.toData(undefined, "value")).toBeUndefined()
expect(Data.toData({ a: undefined }, "value", "result")).toStrictEqual({ a: null })
})
})