codeburn/scripts/perf/lib.mjs

#!/usr/bin/env node
// Shared helpers for the Phase 0 performance harness.
// Extends the #1164 release-acceptance evidence shape without touching product
// hot paths. Isolation matches scripts/upgrade-path/run.mjs cliEnv().

import { spawn } from 'node:child_process'
import { existsSync, mkdirSync, writeFileSync, statSync, readdirSync } from 'node:fs'
import { cpus, homedir, loadavg, release, totalmem, userInfo } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { performance } from 'node:perf_hooks'

export const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..')
export const PERF_DIR = join(REPO, 'perf')
export const RESULTS_DIR = join(PERF_DIR, 'results')

export const TIMINGS_HEADER = [
  'run_id', 'case_id', 'surface', 'persona', 'operation', 'trial', 'cache_state',
  'start_monotonic_ms', 'first_feedback_ms', 'first_useful_ms', 'complete_ms',
  'cpu_peak_pct', 'rss_peak_bytes', 'calls', 'tokens', 'cost_usd', 'identical_totals', 'notes',
].join(',')

export const DESKTOP_OVERVIEW_ARGS = ['status', '--format', 'menubar-json', '--period', 'today', '--no-timeline']
export const MENUBAR_STATUS_ARGS = ['status', '--format', 'menubar-json', '--provider', 'all', '--period', 'today', '--no-optimize']
export const FULL_CORPUS_ARGS = ['status', '--format', 'menubar-json', '--period', 'all', '--no-timeline', '--no-optimize']

export function cliEntry() {
  const dist = join(REPO, 'dist', 'cli.js')
  if (existsSync(dist)) return { cmd: process.execPath, args: [dist], label: 'dist/cli.js' }
  return { cmd: process.execPath, args: ['--import', 'tsx', join(REPO, 'src', 'cli.ts')], label: 'tsx src/cli.ts' }
}

export function isolatedEnv(home, extra = {}) {
  const passthrough = {}
  for (const key of ['PATH', 'PATHEXT', 'SystemRoot', 'ComSpec', 'windir', 'TEMP', 'TMP', 'NUMBER_OF_PROCESSORS', 'NODE_PATH']) {
    if (process.env[key] !== undefined) passthrough[key] = process.env[key]
  }
  const cacheDir = extra.CODEBURN_CACHE_DIR ?? join(home, '.cache', 'codeburn')
  return {
    ...passthrough,
    HOME: home,
    USERPROFILE: home,
    TZ: 'UTC',
    CODEBURN_CACHE_DIR: cacheDir,
    APPDATA: join(home, 'AppData', 'Roaming'),
    LOCALAPPDATA: join(home, 'AppData', 'Local'),
    XDG_CONFIG_HOME: join(home, '.config'),
    XDG_DATA_HOME: join(home, '.local', 'share'),
    XDG_CACHE_HOME: join(home, '.cache'),
    CODEBURN_PRICING_SNAPSHOT_ONLY: '1',
    CODEBURN_FX_NO_FETCH: '1',
    ...extra,
  }
}

export function csvCell(value) {
  const text = value === null || value === undefined ? '' : String(value)
  return /[",\n]/.test(text) ? '"' + text.replaceAll('"', '""') + '"' : text
}

export function percentile(values, p) {
  if (!values.length) return null
  const sorted = [...values].sort((a, b) => a - b)
  const rank = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))
  return sorted[rank]
}

export function stats(values) {
  if (!values.length) return { n: 0, min: null, max: null, mean: null, p50: null, p95: null }
  const sum = values.reduce((a, b) => a + b, 0)
  return {
    n: values.length,
    min: Math.min(...values),
    max: Math.max(...values),
    mean: sum / values.length,
    p50: percentile(values, 50),
    p95: percentile(values, 95),
  }
}

export function runProcess(command, args, options = {}) {
  return new Promise(resolveRun => {
    const started = performance.now()
    const child = spawn(command, args, {
      cwd: options.cwd ?? REPO,
      env: options.env ?? process.env,
      stdio: ['ignore', 'pipe', 'pipe'],
    })
    const stdout = []
    const stderr = []
    child.stdout.on('data', chunk => stdout.push(chunk))
    child.stderr.on('data', chunk => stderr.push(chunk))
    const timeout = options.timeoutMs
      ? setTimeout(() => { child.kill('SIGKILL') }, options.timeoutMs)
      : null
    child.on('error', error => {
      if (timeout) clearTimeout(timeout)
      resolveRun({
        command, args, pid: child.pid ?? null, exitCode: null,
        durationMs: performance.now() - started,
        stdout: '', stderr: String(error),
      })
    })
    child.on('close', exitCode => {
      if (timeout) clearTimeout(timeout)
      resolveRun({
        command, args, pid: child.pid ?? null, exitCode,
        durationMs: performance.now() - started,
        stdout: Buffer.concat(stdout).toString('utf8'),
        stderr: Buffer.concat(stderr).toString('utf8'),
      })
    })
  })
}

export async function capture(command, args, options = {}) {
  const result = await runProcess(command, args, options)
  if (result.exitCode !== 0) {
    throw new Error(command + ' ' + args.join(' ') + ' failed (' + result.exitCode + '): ' + result.stderr.slice(0, 2000))
  }
  return result.stdout.trim()
}

export async function machineSnapshot() {
  const sha = await capture('git', ['rev-parse', 'HEAD'], { cwd: REPO }).catch(() => 'unknown')
  const branch = await capture('git', ['branch', '--show-current'], { cwd: REPO }).catch(() => '')
  const dirty = await capture('git', ['status', '--porcelain=v1'], { cwd: REPO }).catch(() => '')
  // hw.model has no node:os equivalent; everything else does.
  const hardware = await capture('sysctl', ['-n', 'hw.model'], { cwd: REPO }).catch(() => 'unknown')
  return {
    captured_at: new Date().toISOString(),
    sha,
    branch,
    dirty: Boolean(dirty),
    os: process.platform + ' ' + release(),
    arch: process.arch,
    hardware,
    memory_bytes: totalmem(),
    ncpu: cpus().length,
    loadavg: loadavg().map(n => n.toFixed(2)).join(' '),
    node: process.version,
    cli: cliEntry().label,
    cwd: REPO,
    host_home_not_used: true,
  }
}

export async function processRssBytes(pid) {
  if (!Number.isInteger(pid) || pid <= 0) return null
  const result = await runProcess('ps', ['-o', 'rss=', '-p', String(pid)], { timeoutMs: 5_000 })
  if (result.exitCode !== 0) return null
  const kb = Number(result.stdout.trim().split(/\s+/).pop())
  return Number.isFinite(kb) ? kb * 1024 : null
}

export function summarizePayload(output) {
  if (!output) return { calls: '', tokens: '', cost_usd: '' }
  try {
    const payload = JSON.parse(output)
    const current = payload.current ?? payload
    const tokens = Number(current.inputTokens ?? 0) + Number(current.outputTokens ?? 0)
    return {
      calls: current.calls ?? '',
      tokens: Number.isFinite(tokens) ? tokens : '',
      cost_usd: current.cost ?? '',
      sessions: current.sessions ?? '',
      label: current.label ?? '',
      hydrationComplete: payload.hydration ? payload.hydration.complete !== false : true,
    }
  } catch {
    return { calls: '', tokens: '', cost_usd: '', parseError: true }
  }
}

export function writeTimings(path, rows) {
  const body = [TIMINGS_HEADER, ...rows.map(row => TIMINGS_HEADER.split(',').map(key => csvCell(row[key] ?? '')).join(','))].join('\n') + '\n'
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, body)
}

export function fixtureBytes(root) {
  let total = 0
  const walk = dir => {
    if (!existsSync(dir)) return
    for (const name of readdirSync(dir, { withFileTypes: true })) {
      const path = join(dir, name.name)
      if (name.isDirectory()) walk(path)
      else total += statSync(path).size
    }
  }
  walk(root)
  return total
}

export class ServeClient {
  constructor(env, extraArgs = []) {
    this.env = env
    this.extraArgs = extraArgs
    this.child = null
    this.buffer = ''
    this.ready = null
    this.waiters = new Map()
    this.pid = null
    this.stderr = ''
    this._closed = false
  }

  start() {
    const entry = cliEntry()
    this.child = spawn(entry.cmd, [...entry.args, 'serve', '--stdio', ...this.extraArgs], {
      cwd: REPO,
      env: this.env,
      stdio: ['pipe', 'pipe', 'pipe'],
    })
    this.pid = this.child.pid ?? null
    this.ready = new Promise((resolve, reject) => {
      const timer = setTimeout(() => reject(new Error('serve did not emit {ready:true} within 30s')), 30_000)
      this._readyResolve = () => {
        clearTimeout(timer)
        resolve()
      }
      this.child.on('error', err => {
        clearTimeout(timer)
        reject(err)
      })
      this.child.on('exit', (code, signal) => {
        if (!this._closed) {
          clearTimeout(timer)
          reject(new Error('serve exited before ready (code=' + code + ' signal=' + signal + ')'))
        }
      })
    })
    this.child.stdout.setEncoding('utf8')
    this.child.stdout.on('data', chunk => this._onData(chunk))
    this.child.stderr.setEncoding('utf8')
    this.child.stderr.on('data', chunk => { this.stderr += chunk })
    return this.ready
  }

  _onData(chunk) {
    this.buffer += chunk
    let idx
    while ((idx = this.buffer.indexOf('\n')) >= 0) {
      const line = this.buffer.slice(0, idx).trim()
      this.buffer = this.buffer.slice(idx + 1)
      if (!line) continue
      let msg
      try { msg = JSON.parse(line) } catch { continue }
      if (msg.ready) {
        this.pid = typeof msg.pid === 'number' ? msg.pid : this.pid
        this._readyResolve?.()
        continue
      }
      if (typeof msg.progress === 'string' && !('ok' in msg)) {
        const waiter = this.waiters.get(msg.id)
        if (waiter) {
          waiter.progress.push(msg.progress)
          if (waiter.firstFeedbackMs == null) waiter.firstFeedbackMs = performance.now() - waiter.started
        }
        continue
      }
      const waiter = this.waiters.get(msg.id)
      if (waiter) {
        this.waiters.delete(msg.id)
        waiter.finish(msg)
      }
    }
  }

  request(id, args, timeoutMs = 180_000) {
    return new Promise((resolve, reject) => {
      const started = performance.now()
      const entry = {
        started,
        progress: [],
        firstFeedbackMs: null,
        finish: msg => {
          clearTimeout(timer)
          resolve({
            id,
            ok: msg.ok === true,
            refused: msg.refused === true,
            output: typeof msg.output === 'string' ? msg.output : '',
            error: typeof msg.error === 'string' ? msg.error : '',
            progress: entry.progress,
            firstFeedbackMs: entry.firstFeedbackMs,
            completeMs: performance.now() - started,
          })
        },
      }
      const timer = setTimeout(() => {
        this.waiters.delete(id)
        reject(new Error('serve request ' + id + ' timed out after ' + timeoutMs + 'ms'))
      }, timeoutMs)
      this.waiters.set(id, entry)
      this.child.stdin.write(JSON.stringify({ id, args }) + '\n', err => {
        if (err) {
          clearTimeout(timer)
          this.waiters.delete(id)
          reject(err)
        }
      })
    })
  }

  async close() {
    this._closed = true
    if (!this.child) return
    this.child.stdin.end()
    await Promise.race([
      new Promise(resolve => this.child.once('exit', resolve)),
      new Promise(resolve => setTimeout(() => { this.child.kill('SIGKILL'); resolve() }, 5_000)),
    ])
  }
}

export function assertIsolatedHome(home) {
  const resolved = resolve(home)
  const homes = new Set()
  try { homes.add(resolve(homedir())) } catch {}
  try { homes.add(resolve(userInfo().homedir)) } catch {}
  if (process.env.HOME) homes.add(resolve(process.env.HOME))
  for (const realHome of homes) {
    if (resolved === realHome || resolved.startsWith(realHome + '/')) {
      throw new Error('refusing to write fixtures under the real home (' + realHome + ')')
    }
  }
}

export function nowId(prefix = 'perf') {
  return prefix + '-' + new Date().toISOString().replace(/[:.]/g, '-')
}