From a801306ddfffcb5d5592657ff84693c74f3bee9a Mon Sep 17 00:00:00 2001 From: Victor Navarro Date: Sat, 17 Oct 2026 12:18:27 +0200 Subject: [PATCH] chore(infra): remove dead honeycomb monitoring and log processor (#56226) --- bun.lock | 17 -- infra/console.ts | 10 - infra/monitoring.ts | 287 ------------------ infra/secret.ts | 10 - .../app/src/routes/honeycomb/webhook.ts | 111 ------- packages/console/function/package.json | 22 -- .../console/function/src/log-processor.ts | 95 ------ packages/console/function/sst-env.d.ts | 10 - packages/console/function/tsconfig.json | 12 - sst-env.d.ts | 13 - sst.config.ts | 3 - 11 files changed, 590 deletions(-) delete mode 100644 infra/monitoring.ts delete mode 100644 infra/secret.ts delete mode 100644 packages/console/app/src/routes/honeycomb/webhook.ts delete mode 100644 packages/console/function/package.json delete mode 100644 packages/console/function/src/log-processor.ts delete mode 100644 packages/console/function/sst-env.d.ts delete mode 100644 packages/console/function/tsconfig.json diff --git a/bun.lock b/bun.lock index dc78725ae7b7..0845538d2b5a 100644 --- a/bun.lock +++ b/bun.lock @@ -244,21 +244,6 @@ "wrangler": "4.50.0", }, }, - "packages/console/function": { - "name": "@opencode/console-function", - "version": "2.0.32", - "dependencies": { - "@opencode/console-resource": "workspace:*", - }, - "devDependencies": { - "@cloudflare/workers-types": "catalog:", - "@tsconfig/node22": "22.0.2", - "@types/bun": "catalog:", - "@types/node": "catalog:", - "@typescript/native-preview": "catalog:", - "typescript": "catalog:", - }, - }, "packages/console/resource": { "name": "@opencode/console-resource", "dependencies": { @@ -2025,8 +2010,6 @@ "@opencode/console-app": ["@opencode/console-app@workspace:packages/console/app"], - "@opencode/console-function": ["@opencode/console-function@workspace:packages/console/function"], - "@opencode/console-resource": ["@opencode/console-resource@workspace:packages/console/resource"], "@opencode/core": ["@opencode/core@workspace:packages/core"], diff --git a/infra/console.ts b/infra/console.ts index 7e33cd369c07..79d5e077cfac 100644 --- a/infra/console.ts +++ b/infra/console.ts @@ -1,6 +1,5 @@ import { domain } from "./stage" import { EMAILOCTOPUS_API_KEY } from "./app" -import { SECRET } from "./secret" // Preview stages have no new Console or inference deployment to forward to. const migrationDomain = @@ -20,7 +19,6 @@ const consoleMigration = new sst.Linkable("ConsoleMigration", { new sst.cloudflare.Bucket("ZenData") new sst.cloudflare.Bucket("ZenDataNew") -const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL") const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID") const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY") const ENTERPRISE_SALES_INBOX_EMAIL = new sst.Secret("ENTERPRISE_SALES_INBOX_EMAIL") @@ -33,18 +31,11 @@ const CONSOLE_CRM_LOOKUP_URL = new sst.Linkable("CONSOLE_CRM_LOOKUP_URL", { properties: { value: `https://${domain}/console/api/internal/console-account` }, }) -const logProcessor = new sst.cloudflare.Worker("LogProcessor", { - handler: "packages/console/function/src/log-processor.ts", - link: [SECRET.HoneycombApiKey], -}) - new sst.cloudflare.x.SolidStart("Console", { domain, path: "packages/console/app", link: [ consoleMigration, - DISCORD_INCIDENT_WEBHOOK_URL, - SECRET.HoneycombWebhookSecret, EMAILOCTOPUS_API_KEY, AWS_SES_ACCESS_KEY_ID, AWS_SES_SECRET_ACCESS_KEY, @@ -75,7 +66,6 @@ new sst.cloudflare.x.SolidStart("Console", { ...(bindings ?? []), { type: "service", name: "INFERENCE", service: `opencode-inference-${$app.stage}` }, ]) - args.tailConsumers = [{ service: logProcessor.nodes.worker.scriptName }] }, }, }, diff --git a/infra/monitoring.ts b/infra/monitoring.ts deleted file mode 100644 index b5521d61ce24..000000000000 --- a/infra/monitoring.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { SECRET } from "./secret" -import { domain } from "./stage" - -const description = "Managed by SST (Don't edit in Honeycomb UI)" -const alertsDisabled = $app.stage !== "production" - -const webhookRecipient = new honeycombio.WebhookRecipient("DiscordAlerts", { - name: $app.stage === "production" ? "Discord Alerts" : `Discord Alerts (${$app.stage})`, - url: `https://${domain}/honeycomb/webhook`, - secret: SECRET.HoneycombWebhookSecret.result, - templates: [ - { - type: "trigger", - body: `{ - "url": {{ .Result.URL | quote }}, - "type": {{ .Vars.type | quote }}, - "name": {{ .Name | quote }}, - "status": {{ .Alert.Status | quote }}, - "isTest": {{ .Alert.IsTest }}, - "groups": {{ .Result.GroupsTriggered | toJson }} - }`, - }, - ], - variables: [ - { - name: "type", - }, - ], -}) - -// Honeycomb can keep stale query-local calculated fields when the name is unchanged, -// so tie the field name to the expression while avoiding deploy-to-deploy churn. -// https://github.com/honeycombio/terraform-provider-honeycombio/issues/852 -const calculatedField = (field: { name: string; expression: string }) => ({ - ...field, - name: `${field.name}_${( - Array.from(field.expression).reduce((result, char) => Math.imul(31, result) + char.charCodeAt(0), 0) >>> 0 - ).toString(36)}`, -}) - -const modelHttpErrorsQuery = (product: "go" | "zen") => { - const filters = [ - { column: "model", op: "exists" }, - { column: "event_type", op: "=", value: "completions" }, - { column: "user_agent", op: "contains", value: "opencode" }, - { column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" }, - ] - const failedHttpStatus = calculatedField({ - name: "is_failed_http_status", - expression: ` -IF( - AND( - GTE($status, "400"), - NOT(EQUALS($status, "401")), - NOT( - AND( - EQUALS($status, "429"), - OR( - EQUALS($error.type, "GoUsageLimitError"), - EQUALS($error.type, "FreeUsageLimitError") - ) - ) - ) - ), - 1, - 0 -)`, - }) - - return honeycombio.getQuerySpecificationOutput({ - breakdowns: ["model"], - calculatedFields: [failedHttpStatus], - calculations: [ - { op: "COUNT", name: "TOTAL", filterCombination: "AND", filters }, - { - op: "SUM", - name: "FAILED", - column: failedHttpStatus.name, - filterCombination: "AND", - filters, - }, - ], - formulas: [{ name: "ERROR", expression: "IF(GTE($TOTAL, 150), DIV($FAILED, $TOTAL), 0)" }], - timeRange: 900, - }).json -} - -const providerHttpErrorsQuery = () => { - const filters = [ - { column: "provider", op: "exists" }, - { column: "user_agent", op: "contains", value: "opencode" }, - ] - const successHttpStatus = calculatedField({ - name: "is_success_http_status", - expression: `IF(AND(GTE($status, "200"), LT($status, "400")), 1, 0)`, - }) - const failedProviderHttpStatus = calculatedField({ - name: "is_failed_provider_http_status", - expression: `IF(GT($llm.error.code, "400"), 1, 0)`, - }) - - return honeycombio.getQuerySpecificationOutput({ - breakdowns: ["provider"], - calculatedFields: [successHttpStatus, failedProviderHttpStatus], - calculations: [ - { - op: "SUM", - name: "SUCCESS", - column: successHttpStatus.name, - filterCombination: "AND", - filters: [...filters, { column: "event_type", op: "=", value: "completions" }], - }, - { - op: "SUM", - name: "FAILED", - column: failedProviderHttpStatus.name, - filterCombination: "AND", - filters: [ - ...filters, - { column: "event_type", op: "=", value: "llm.error" }, - { column: "llm.error.code", op: "!=", value: "404" }, - ], - }, - ], - formulas: [ - { name: "ERROR", expression: "IF(GTE(SUM($SUCCESS, $FAILED), 150), DIV($FAILED, SUM($SUCCESS, $FAILED)), 0)" }, - ], - timeRange: 900, - }).json -} - -const modelLowTpsQuery = (product: "go" | "zen") => { - const filters = [ - { column: "model", op: "exists" }, - { column: "event_type", op: "=", value: "completions" }, - { column: "user_agent", op: "contains", value: "opencode" }, - { column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" }, - { column: "status", op: ">=", value: "200" }, - { column: "status", op: "<", value: "400" }, - { column: "tps.output", op: "exists" }, - ] - - return honeycombio.getQuerySpecificationOutput({ - breakdowns: ["model"], - calculations: [ - { op: "COUNT", name: "TOTAL", filterCombination: "AND", filters }, - { - op: "P50", - name: "TPS", - column: "tps.output", - filterCombination: "AND", - filters, - }, - ], - formulas: [{ name: "LOW_TPS", expression: "IF(GTE($TOTAL, 100), $TPS, 999)" }], - timeRange: 1800, - }).json -} - -new honeycombio.Trigger("IncreasedModelHttpErrorsGo", { - name: "Increased Model HTTP Errors [Go]", - description, - disabled: alertsDisabled, - queryJson: modelHttpErrorsQuery("go"), - alertType: "on_change", - frequency: 300, - thresholds: [{ op: ">=", value: 0.7, exceededLimit: 1 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "model_http_errors" }], - }, - ], - }, - ], -}) - -new honeycombio.Trigger("IncreasedModelHttpErrorsZen", { - name: "Increased Model HTTP Errors [Zen]", - description, - disabled: alertsDisabled, - queryJson: modelHttpErrorsQuery("zen"), - alertType: "on_change", - frequency: 300, - thresholds: [{ op: ">=", value: 0.7, exceededLimit: 1 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "model_http_errors" }], - }, - ], - }, - ], -}) - -new honeycombio.Trigger("LowModelTpsGo", { - name: "Low Model TPS [Go]", - description, - disabled: alertsDisabled, - queryJson: modelLowTpsQuery("go"), - alertType: "on_change", - frequency: 600, - thresholds: [{ op: "<=", value: 10, exceededLimit: 1 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "model_low_tps" }], - }, - ], - }, - ], -}) - -new honeycombio.Trigger("LowModelTpsZen", { - name: "Low Model TPS [Zen]", - description, - disabled: alertsDisabled, - queryJson: modelLowTpsQuery("zen"), - alertType: "on_change", - frequency: 600, - thresholds: [{ op: "<=", value: 10, exceededLimit: 1 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "model_low_tps" }], - }, - ], - }, - ], -}) - -new honeycombio.Trigger("IncreasedProviderHttpErrors", { - name: "Increased Provider HTTP Errors", - description, - disabled: alertsDisabled, - queryJson: providerHttpErrorsQuery(), - alertType: "on_change", - frequency: 300, - thresholds: [{ op: ">=", value: 0.7, exceededLimit: 1 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "provider_http_errors" }], - }, - ], - }, - ], -}) - -new honeycombio.Trigger("IncreasedFreeTierRequests", { - name: "Increased Free Tier Requests", - description, - disabled: alertsDisabled, - queryJson: honeycombio.getQuerySpecificationOutput({ - calculations: [{ op: "COUNT" }], - filters: [ - { column: "event_type", op: "=", value: "completions" }, - { column: "user_agent", op: "contains", value: "opencode" }, - { column: "isFreeTier", op: "=", value: "true" }, - ], - timeRange: 3600, - }).json, - alertType: "on_change", - frequency: 900, - thresholds: [{ op: ">=", value: 50, exceededLimit: 1 }], - baselineDetails: [{ type: "percentage", offsetMinutes: 1440 }], - recipients: [ - { - id: webhookRecipient.id, - notificationDetails: [ - { - variables: [{ name: "type", value: "custom" }], - }, - ], - }, - ], -}) diff --git a/infra/secret.ts b/infra/secret.ts deleted file mode 100644 index e32ee6bd4cf5..000000000000 --- a/infra/secret.ts +++ /dev/null @@ -1,10 +0,0 @@ -sst.Linkable.wrap(random.RandomPassword, (resource) => ({ - properties: { - value: resource.result, - }, -})) - -export const SECRET = { - HoneycombApiKey: new sst.Secret("HONEYCOMB_API_KEY"), - HoneycombWebhookSecret: new random.RandomPassword("HoneycombWebhookSecret", { length: 24 }), -} diff --git a/packages/console/app/src/routes/honeycomb/webhook.ts b/packages/console/app/src/routes/honeycomb/webhook.ts deleted file mode 100644 index bbb9d23dbf53..000000000000 --- a/packages/console/app/src/routes/honeycomb/webhook.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { APIEvent } from "@solidjs/start/server" -import { z } from "zod" -import { Resource } from "@opencode/console-resource" -import { timingSafeEqual } from "node:crypto" - -const DISCORD_ALERT_ROLE_ID = "1520924666359713863" - -const basePayload = z.object({ - name: z.string().optional(), - status: z.string().optional(), - isTest: z.boolean().optional(), - url: z.string(), -}) - -const groups = z - .object({ - result: z.union([z.number(), z.string()]).nullish(), - group: z.object({ key: z.string(), value: z.string() }).array(), - }) - .array() - -const honeycombWebhookPayload = z.discriminatedUnion("type", [ - basePayload.extend({ - type: z.literal("model_http_errors"), - groups, - }), - basePayload.extend({ - type: z.literal("model_low_tps"), - groups, - }), - basePayload.extend({ - type: z.literal("provider_http_errors"), - groups, - }), - basePayload.extend({ - type: z.literal("custom"), - }), -]) - -const postDiscordMessage = async (payload: z.infer) => { - const names = - payload.type === "custom" - ? [] - : payload.groups.flatMap((item) => - item.group.map((g) => { - const result = item.result == null ? undefined : Number(item.result) - return `- ${g.value}${ - result !== undefined && Number.isFinite(result) - ? payload.type === "model_low_tps" - ? ` (${Math.round(result)} TPS)` - : ` (${Math.round(result * 100)}% errors)` - : "" - }` - }), - ) - - const content = [ - `[**${payload.isTest ? "[TEST] " : ""}${payload.name ?? "Honeycomb alert"}**](${payload.url})`, - ...names, - "", - `<@&${DISCORD_ALERT_ROLE_ID}>`, - ] - .filter((line) => line !== undefined) - .join("\n") - - return fetch(Resource.DISCORD_INCIDENT_WEBHOOK_URL.value, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - content, - allowed_mentions: { roles: [DISCORD_ALERT_ROLE_ID] }, - flags: 4, - }), - }) -} - -export async function POST(input: APIEvent) { - const token = input.request.headers.get("X-Honeycomb-Webhook-Token") - if (!safeEqual(token ?? "", Resource.HoneycombWebhookSecret.value)) { - console.debug("Invalid Honeycomb webhook token") - return Response.json({ message: "invalid token" }, { status: 401 }) - } - - const body = await input.request.json() - console.log(body, JSON.stringify(body, null, 2)) - - const parsed = honeycombWebhookPayload.safeParse(body) - - if (!parsed.success) { - console.error(parsed.error) - return Response.json({ message: "invalid payload" }, { status: 400 }) - } - - if (parsed.data.status !== "TRIGGERED") { - console.debug("Skipping resolved alert Honeycomb webhook") - return Response.json({ message: "ignored" }, { status: 200 }) - } - - const response = await postDiscordMessage(parsed.data) - if (!response.ok) { - return Response.json({ message: "discord webhook failed" }, { status: 502 }) - } - - return Response.json({ message: "sent" }, { status: 200 }) -} - -function safeEqual(a: string, b: string) { - const aBytes = new TextEncoder().encode(a) - const bBytes = new TextEncoder().encode(b) - return aBytes.length === bBytes.length && timingSafeEqual(aBytes, bBytes) -} diff --git a/packages/console/function/package.json b/packages/console/function/package.json deleted file mode 100644 index 059a9cf82273..000000000000 --- a/packages/console/function/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "@opencode/console-function", - "version": "2.0.32", - "$schema": "https://json.schemastore.org/package.json", - "private": true, - "type": "module", - "license": "MIT", - "scripts": { - "typecheck": "tsgo --noEmit" - }, - "devDependencies": { - "@cloudflare/workers-types": "catalog:", - "@tsconfig/node22": "22.0.2", - "@types/bun": "catalog:", - "@types/node": "catalog:", - "typescript": "catalog:", - "@typescript/native-preview": "catalog:" - }, - "dependencies": { - "@opencode/console-resource": "workspace:*" - } -} diff --git a/packages/console/function/src/log-processor.ts b/packages/console/function/src/log-processor.ts deleted file mode 100644 index ede67ac6f35d..000000000000 --- a/packages/console/function/src/log-processor.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { TraceItem } from "@cloudflare/workers-types" - -export default { - async tail(events: TraceItem[]) { - for (const event of events) { - if (!event.event) continue - if (!("request" in event.event)) continue - if (event.event.request.method !== "POST") continue - - const url = new URL(event.event.request.url) - if ( - url.pathname !== "/zen/v1/chat/completions" && - url.pathname !== "/zen/v1/messages" && - url.pathname !== "/zen/v1/responses" && - !url.pathname.startsWith("/zen/v1/models/") && - url.pathname !== "/zen/go/v1/chat/completions" && - url.pathname !== "/zen/go/v1/messages" && - url.pathname !== "/zen/go/v1/responses" && - !url.pathname.startsWith("/zen/go/v1/models/") - ) - continue - - const ip = event.event.request.headers["x-real-ip"] - let data: Record = { - "cf.continent": event.event.request.cf?.continent, - "cf.country": event.event.request.cf?.country, - "cf.city": event.event.request.cf?.city, - "cf.region": event.event.request.cf?.region, - "cf.latitude": event.event.request.cf?.latitude, - "cf.longitude": event.event.request.cf?.longitude, - "cf.timezone": event.event.request.cf?.timezone, - duration: event.wallTime, - request_length: parseInt(event.event.request.headers["content-length"] ?? "0"), - status: event.event.response?.status ?? 0, - ip, - "ip.prefix": ipPrefix(ip), - } - const time = new Date(event.eventTimestamp ?? Date.now()).toISOString() - // This also merges each _metric log into data, so keep it while the Honeycomb export below is disabled. - const events = [ - ...event.logs.flatMap((log) => - log.message.flatMap((message: string) => { - if (!message.startsWith("_metric:")) return [] - const json = JSON.parse(message.slice(8)) as Record - data = { ...data, ...json } - if ("llm.error.code" in json) { - return [{ time, data: { ...data, event_type: "llm.error" } }] - } - return [] - }), - ), - { time, data: { ...data, event_type: "completions" } }, - ] - console.log(JSON.stringify(data, null, 2)) - void events - - // const honeycomb = await fetch("https://api.honeycomb.io/1/batch/zen", { - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value, - // }, - // body: JSON.stringify(events), - // }) - // console.log(honeycomb.status) - // console.log(await honeycomb.text()) - } - }, -} - -// Returns a stable lookup key for an IP address. -// IPv4: full address as /32 (e.g. "203.0.113.45/32"). -// IPv6: the /64 network prefix (e.g. "2001:db8:abcd:1234::/64"). ISPs commonly -// rotate the lower 64 host bits via SLAAC privacy extensions (RFC 8981), so -// grouping by /64 collapses those rotations into one key. -function ipPrefix(ip: string | undefined) { - if (!ip) return undefined - if (ip.includes(".") && !ip.includes(":")) return `${ip}/32` - if (!ip.includes(":")) return undefined - - // Expand "::" to its full form, then keep the first 4 hextets. - const [head, tail] = ip.split("::") as [string, string | undefined] - const headParts = head ? head.split(":") : [] - const tailParts = tail !== undefined ? tail.split(":") : [] - const missing = 8 - headParts.length - tailParts.length - if (missing < 0) return undefined - const full = [...headParts, ...new Array(missing).fill("0"), ...tailParts] - if (full.length !== 8) return undefined - - const prefix = full - .slice(0, 4) - .map((part) => part.toLowerCase().replace(/^0+(?=.)/, "")) - .join(":") - return `${prefix}::/64` -} diff --git a/packages/console/function/sst-env.d.ts b/packages/console/function/sst-env.d.ts deleted file mode 100644 index 301538ccb214..000000000000 --- a/packages/console/function/sst-env.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* This file is auto-generated by SST. Do not edit. */ -/* tslint:disable */ -/* eslint-disable */ -/* deno-fmt-ignore-file */ -/* biome-ignore-all lint: auto-generated */ - -/// - -import "sst" -export {} \ No newline at end of file diff --git a/packages/console/function/tsconfig.json b/packages/console/function/tsconfig.json deleted file mode 100644 index 17564281e949..000000000000 --- a/packages/console/function/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "@tsconfig/node22/tsconfig.json", - "compilerOptions": { - "noUnusedLocals": true, - "module": "ESNext", - "moduleResolution": "bundler", - "jsx": "preserve", - "jsxImportSource": "react", - "types": ["@cloudflare/workers-types", "bun", "node"] - } -} diff --git a/sst-env.d.ts b/sst-env.d.ts index b331a38f9c7e..bcfa83ba237d 100644 --- a/sst-env.d.ts +++ b/sst-env.d.ts @@ -24,10 +24,6 @@ declare module "sst" { "type": "sst.cloudflare.SolidStart" "url": string } - "DISCORD_INCIDENT_WEBHOOK_URL": { - "type": "sst.sst.Secret" - "value": string - } "DISCORD_SUPPORT_BOT_TOKEN": { "type": "sst.sst.Secret" "value": string @@ -57,15 +53,6 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } - "HONEYCOMB_API_KEY": { - "type": "sst.sst.Secret" - "value": string - } - "HoneycombWebhookSecret": { - "type": "random.index/randomPassword.RandomPassword" - "value": string - } - "LogProcessor": import("@cloudflare/workers-types").Service "SALESFORCE_CLIENT_ID": { "type": "sst.sst.Secret" "value": string diff --git a/sst.config.ts b/sst.config.ts index 91c174dd2230..84f56b653601 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -41,9 +41,6 @@ export default $config({ const stats = stage.deployAws ? await import("./infra/stats.js") : undefined await import("./infra/console.js") await import("./infra/share.js") - if ($app.stage === "production" || $app.stage === "vimtor") { - await import("./infra/monitoring.js") - } return { ...(stats ? { StatsUrl: stats.app.url } : {}),