mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-21 14:12:28 +00:00
Merge pull request #1522: release 7.6.0 blockers (Windows direct restart, Project card in Main, CI test repairs, no scheduled CI runs)
Release 7.6.0 blockers: Windows direct restart, Project card in Main, CI test repairs, no scheduled CI runs
This commit is contained in:
Ouroboros
GitHub
commit
8d5142374a
24 changed files with 346 additions and 158 deletions
34
.github/workflows/ci.yml
vendored
34
.github/workflows/ci.yml
vendored
|
|
@ -5,7 +5,7 @@
|
|||
# Tier 2.5 (Integration): Manual / tag (branch pushes: provider-canary-push.yml) → Real-provider tests, reusable provider-canary.yml (~2 min)
|
||||
# Tier 2.6 (Skill smoke): Push to ouroboros-stable / manual / tag → LIVE OuroborosHub official-skill install smoke (3-OS, ~5 min) + review→grant→enable-persistence flow on one cheap reviewer slot (ubuntu-only step, OPENROUTER_API_KEY, ~$1.2/run)
|
||||
# Tier 3 (Build+Release): Tag v* → PyInstaller + GitHub Release (~15 min)
|
||||
# Scheduled (System E2E): Daily cron / manual → KEYLESS tests/system_e2e/ scenario lane (Ubuntu, ≤40 min)
|
||||
# Opt-in (System E2E): Manual / tag (no schedule) → KEYLESS tests/system_e2e/ scenario lane (Ubuntu, ≤40 min)
|
||||
# Manual (Live E2E): Dispatch input e2e_live=true only (no schedule) → PAID devtools/e2e_live stand, $30 cap, OUROBOROS_E2E_LIVE_OPENROUTER_KEY (skipped honestly without it)
|
||||
#
|
||||
# Tier 2.5 requires OPENROUTER_API_KEY / OPENAI_API_KEY / ANTHROPIC_API_KEY.
|
||||
|
|
@ -41,16 +41,10 @@ on:
|
|||
description: 'Run the paid live E2E stand ($30 cap; needs the OUROBOROS_E2E_LIVE_OPENROUTER_KEY secret)'
|
||||
type: boolean
|
||||
default: false
|
||||
# One cron, owned by exactly ONE job through `github.event.schedule`: 04:37
|
||||
# is the keyless system-e2e-mock lane (the paid e2e-live stand runs only on
|
||||
# its opt-in dispatch input). Off the hour deliberately: GitHub's cron queue
|
||||
# is deepest at :00. NOTE for any job added later — a scheduled run reports
|
||||
# the DEFAULT BRANCH in github.ref, so a job gated on a branch ref alone fires
|
||||
# on the schedule (integration-test carries the explicit event guard), and a
|
||||
# job gated on the bare event name fires on every cron this block carries —
|
||||
# bind a scheduled job to its own cron string.
|
||||
schedule:
|
||||
- cron: '37 4 * * *'
|
||||
# No schedule (owner, 2026-10-05): nobody read the nightly results. The
|
||||
# keyless system-e2e-mock lane runs on manual dispatch and release tags. A
|
||||
# scheduled run would report the DEFAULT BRANCH in github.ref, which is why
|
||||
# the paid lanes below keep their `!= 'schedule'` guards.
|
||||
|
||||
# Read-only is sufficient for tests and fork PRs. The release job overrides
|
||||
# this narrowly with contents: write when publishing a tag.
|
||||
|
|
@ -367,9 +361,8 @@ jobs:
|
|||
# ──────────────────────────────────────────────────────────────────
|
||||
integration-test:
|
||||
# A scheduled run is neither a manual run nor a tag. The leading guard keeps
|
||||
# the nightly cron above OFF this paid lane even if a branch-ref condition
|
||||
# joins the group below: a scheduled run carries the default branch in
|
||||
# github.ref.
|
||||
# any schedule OFF this paid lane even if a branch-ref condition joins the
|
||||
# group below: a scheduled run carries the default branch in github.ref.
|
||||
if: |
|
||||
github.event_name != 'schedule'
|
||||
&& (github.event_name == 'workflow_dispatch'
|
||||
|
|
@ -586,12 +579,12 @@ jobs:
|
|||
"python -m pytest tests/ -m portable_detail -q --tb=short"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────
|
||||
# Scheduled: the KEYLESS system_e2e scenario lane (tests/system_e2e/).
|
||||
# The KEYLESS system_e2e scenario lane (tests/system_e2e/).
|
||||
# Those tests carry `integration` + `serial` AND the OUROBOROS_E2E_DEEP env
|
||||
# gate, so no other job in this file can reach them — without this one the
|
||||
# suite would be proof nobody executes. Deliberately NOT a push/PR lane
|
||||
# (owner 9A, replacing the plan §8 PR lane): each scenario spawns a real
|
||||
# isolated server, so the cost belongs on a nightly, not on every review.
|
||||
# isolated server, so it runs on manual dispatch and release tags only.
|
||||
# No provider secrets: the lane is keyless by construction and this job
|
||||
# names none, which is the only way a job gets one. The four OUROBOROS_*
|
||||
# roots point into runner.temp so no scenario can reach a real install.
|
||||
|
|
@ -599,13 +592,8 @@ jobs:
|
|||
# ──────────────────────────────────────────────────────────────────
|
||||
system-e2e-mock:
|
||||
# Also on a release tag: the release bar must not skip the 57-scenario lane
|
||||
# (owner decision, batch №13 item 4, 2026-09-02). Bound to ITS cron string,
|
||||
# so a cron another job adds to the schedule block never runs this lane a
|
||||
# second time each night.
|
||||
if: |
|
||||
(github.event_name == 'schedule' && github.event.schedule == '37 4 * * *')
|
||||
|| github.event_name == 'workflow_dispatch'
|
||||
|| startsWith(github.ref, 'refs/tags/v')
|
||||
# (owner decision, batch №13 item 4, 2026-09-02).
|
||||
if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -214,7 +214,8 @@ Upload the manifest, index, lane results, screenshots and each lane's
|
|||
`traces/` bundle even on failure, never a lane's `data/` tree; read a run
|
||||
with `gh run download <run-id> -n e2e-live-run`. The summary renders
|
||||
verdicts or the typed refusal/error without changing the stand's exit
|
||||
verdict. Browser PR proof and the keyless system-E2E schedule retain their
|
||||
verdict. Browser PR proof and the keyless system-E2E lane (manual dispatch and
|
||||
release tags) retain their
|
||||
separate existing CI owners; the latter uploads its scenario servers'
|
||||
`data/logs/`, `data/task_results/` and the journal segments rotated into
|
||||
`data/archive/*.jsonl` as `system-e2e-traces`. Both uploads are
|
||||
|
|
|
|||
|
|
@ -343,4 +343,4 @@ Settings save classifies effects into three classes rather than claiming everyth
|
|||
|
||||
A visible change is exercised in at least one relevant real consumer flow and the rendered result is inspected with vision; a saved screenshot alone is not verification. Mobile, WebKit, additional browsers and special viewports are selected from the actual interaction risk — not a universal matrix (reviewer items: `docs/CHECKLISTS.md`). No visual-QA runner, endpoint, ledger, or mandatory device matrix is introduced by this policy.
|
||||
|
||||
History adds an inert `task_evidence` carrier for represented or currently addressed tasks without narration. It spends no conversation quota, supplies no physical row identity or liveness, and preserves read gaps or known missing task calls even with zero observations. Ordinary archive/window bounds retain provenance without creating tool activity or changing known invocation outcomes. Invocation identity merges replay and late live settlement; one request parses each tool-log window and lists `archive/` once, replaying that listing's gap to every task (`replay_evidence_for_tasks`). A carrier keeps the whole terminal truth: when it is the task's only terminal row in the window, reload settles the card (warn chip, cause, cost, reviews, Continue) from it. Quiz-bound `owner_wait` facts project independently of optional Project question details, so Main and Project waits end on the same recorded answer/resume facts. Confirmed pauses leave task-keyed System timeline rows through `pause_notices`; local persistence is separate from best-effort external delivery and preserves current Telegram routing.
|
||||
History adds an inert `task_evidence` carrier for represented or currently addressed tasks without narration, in the task's own room only: a notice or lifecycle row Main pins for a Project task mints no carrier, and so no card, in Main. It spends no conversation quota, supplies no physical row identity or liveness, and preserves read gaps or known missing task calls even with zero observations. Ordinary archive/window bounds retain provenance without creating tool activity or changing known invocation outcomes. Invocation identity merges replay and late live settlement; one request parses each tool-log window and lists `archive/` once, replaying that listing's gap to every task (`replay_evidence_for_tasks`). A carrier keeps the whole terminal truth: when it is the task's only terminal row in the window, reload settles the card (warn chip, cause, cost, reviews, Continue) from it. Quiz-bound `owner_wait` facts project independently of optional Project question details, so Main and Project waits end on the same recorded answer/resume facts. Confirmed pauses leave task-keyed System timeline rows through `pause_notices`; local persistence is separate from best-effort external delivery and preserves current Telegram routing.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ The local `ouroboros-stable` ref is also a recovery fallback maintained by expli
|
|||
| Android source/build proof | `android-test` | Every non-scheduled run; `android/tests`, release tests and disposable-key APK compilation; no publisher secrets; no physical proof. |
|
||||
| Android emulator proof | `android-emulator-smoke` | Tags or verified `android/` changes from fork-safe `android-test` base/head comparison; API 26/29/30/33/36 x86_64 install/start and same-UID PackageInstaller readback; no publisher secrets; no physical proof. |
|
||||
| Browser consumer proof | `ui-browser.yml`, called by `ci.yml` and `ui-browser-push.yml` | Reusable complete lane with no input: PRs, manual runs and `v*` tags via `ci.yml`; every `ouroboros` push and a manual `full` run via `ui-browser-push.yml`, whose other manual selections run one PARTIAL DIAGNOSTIC scenario in that workflow's own job, outside the lane. No path filter: Makefile-, spec- and requirements-only merges run too. Full `ui_browser` lane as four `ui-shard` jobs (every fourth sorted node id, each with the engine-availability, whole-lane collection and executed-node guards), one unsharded `ui-manifest` collection and the `ui-smoke` verdict, which reconciles the shards' evidence against the manifest and refuses a red session; manual runs and tags also keep browser-tool coverage, on shard 1. `scripts/browser_ci_scope.py` skips only a complete-documentation diff; an unreadable base runs the lane. No schedule, no new cron, no provider secrets, no added default local pytest lane. |
|
||||
| Scheduled and opt-in lanes | `system-e2e-mock` (keyless; daily 04:37 UTC, manual runs, `v*` tags — on the release bar), `e2e-live` (`OUROBOROS_E2E_LIVE_OPENROUTER_KEY`, `$30` cap) | `e2e-live` runs on no schedule: only a dispatch that opts in through the `e2e_live` input starts it, seeding the dispatched sha; without the secret it is one summary line and green, never a pretend run. Both upload their servers' journals on every outcome, never a settings file: `system-e2e-traces`, and each lane's key-redacted `traces/` in `e2e-live-run` |
|
||||
| Opt-in lanes, no schedule | `system-e2e-mock` (keyless; manual runs and `v*` tags — on the release bar), `e2e-live` (`OUROBOROS_E2E_LIVE_OPENROUTER_KEY`, `$30` cap) | `e2e-live` runs on no schedule: only a dispatch that opts in through the `e2e_live` input starts it, seeding the dispatched sha; without the secret it is one summary line and green, never a pretend run. Both upload their servers' journals on every outcome, never a settings file: `system-e2e-traces`, and each lane's key-redacted `traces/` in `e2e-live-run` |
|
||||
| Ordinary desktop tests | `full-test` (no secrets) | Every PR into `ouroboros`, and in this repository every `ouroboros` push (no path filter), runs Windows/macOS alongside Ubuntu quick-test, so the landed commit shows which merge broke a desktop system; stable pushes, manual runs and `v*` tags retain the full three-OS matrix |
|
||||
| Trusted skill matrix | `skill-smoke` (`OPENROUTER_API_KEY`) | `ouroboros-stable` pushes, manual runs, `v*` tags — never pull requests |
|
||||
| Trusted provider run | `integration-test`, body in `provider-canary.yml` | provider secrets; manual runs and `v*` tags via `ci.yml` (the release chain needs it), `main`/`ouroboros`/`ouroboros-stable` pushes via `provider-canary-push.yml` (its own path list, so a prompts-/skills-only push starts no canary run; its own "Provider canaries" result, so a provider outage does not redden `ci.yml`) — never pull requests |
|
||||
|
|
@ -40,7 +40,7 @@ Dev-facing release/review scripts: `scripts/safe_test.py` (stdlib verification b
|
|||
|
||||
The deep-integration suite drives a REAL `server.py` per scenario — an isolated repo clone, data root and free loopback port through `KeylessIsolatedServer`, whose child environment can never carry a provider credential — and asserts DURABLE artifacts through `ArtifactOracle` readers (`task_results/`, `logs/*.jsonl`, `state/*`), never an HTTP 200 or a harness exit code. Synchronization is durable-event polling (`wait_until`), never bare sleeps; a scenario that must act MID-ROUND holds the loopback model on an event-gated `ModelGate` instead of racing a latency sleep. Two loopback models share one HTTP base (JSON for non-streaming, complete SSE framing for streaming): `ScriptedStubModel`, an ordered per-scenario tool script whose review-organ calls are classified by prompt markers and answered with canned parse-clean verdicts, and `ReplayModel`, deterministic fixtures bound by `(lineage, slot, attempt)` where a miss or an unconsumed row is red. A `ReviewScript` overrides the canned verdicts for exactly the scripted red rounds, and review calls never consume agent script steps. The prompt markers are VERBATIM literals grepped back out of their source files by a default-lane pin, so drift is a named failure rather than a silently mute stub. Reviewer routing is pinned through the structured `OUROBOROS_REVIEWER_SLOTS`, because the retired comma keys are dropped at load and would fall back to the shipped paid default panel. `FakeClaudexorDaemon` imitates the exact `gateways/claudexor.py` client contract — authenticated handshake reporting the TREE'S OWN runtime-pin identity, capability/quota answers, idempotency-keyed project registration and run creation with the engine's replay semantics, run detail and cancel — and records every wire request for scenario assertions.
|
||||
|
||||
The scenario inventory is DATA (`SCENARIOS` in `harness.py`) with a two-direction pin over every `test_*.py` module of the package: a manifest row without a test is red, and a scenario test without a manifest row is red. It covers boot and identity, the review organs (commit triad plus scope in both enforcement classes, plan review, the acceptance loop), egress hardening, typed tool safety, honest cost names, subagent trees, cancellation with its cascade and the owner stop of a direct-chat turn, managed update (fast-forward, carrier transfer, conflicting refusal, crash mid-apply, rollback), delegated transport including the mutating pull-in and its refusal, and the whole skills lifecycle. Scenario tests carry BOTH the `integration` and `serial` markers plus the `OUROBOROS_E2E_DEEP=mock` env gate, so neither the default local run nor either CI pytest pass executes them; the keyless lane is `OUROBOROS_E2E_DEEP=mock pytest tests/system_e2e/ -o addopts=""`, and in CI it is the secret-free `system-e2e-mock` job — daily schedule, manual dispatch and `v*` tags, where `release-preflight` needs it. The paid owner-shaped stand that runs real servers against a real provider is operator tooling: `devtools/e2e_live/README.md`.
|
||||
The scenario inventory is DATA (`SCENARIOS` in `harness.py`) with a two-direction pin over every `test_*.py` module of the package: a manifest row without a test is red, and a scenario test without a manifest row is red. It covers boot and identity, the review organs (commit triad plus scope in both enforcement classes, plan review, the acceptance loop), egress hardening, typed tool safety, honest cost names, subagent trees, cancellation with its cascade and the owner stop of a direct-chat turn, managed update (fast-forward, carrier transfer, conflicting refusal, crash mid-apply, rollback), delegated transport including the mutating pull-in and its refusal, and the whole skills lifecycle. Scenario tests carry BOTH the `integration` and `serial` markers plus the `OUROBOROS_E2E_DEEP=mock` env gate, so neither the default local run nor either CI pytest pass executes them; the keyless lane is `OUROBOROS_E2E_DEEP=mock pytest tests/system_e2e/ -o addopts=""`, and in CI it is the secret-free `system-e2e-mock` job — manual dispatch and `v*` tags, where `release-preflight` needs it. The paid owner-shaped stand that runs real servers against a real provider is operator tooling: `devtools/e2e_live/README.md`.
|
||||
|
||||
### Build scripts
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Close stops this generation’s workers/services and releases its PID lock, reta
|
|||
|
||||
Planned restart retains same-pin daemons. After worker custody, before optional cleanup, graceful and final paths call `_stop_owned_daemon_for_new_pin`: `load_runtime_pin` versus attach-only `read_owned_gateway`. Unpublished/unreadable pins and unreachable/foreign/stopped daemons leave handoff unchanged; unconfirmed stop retains custody/diagnostics for reconciliation without invented spend.
|
||||
|
||||
Teardown sets the stop flag, joins the supervisor within its bound and terminalizes `kill_workers` before optional waits/sweeps. SIGKILL may lose writes/late frames; snapshot restore recovers missed writes. Queue init hands over `state/direct_roots.json` before clearing it, fencing listed roots like surviving RUNNING rows as Cancelled, not infra-failed. The restart watcher owns final worker/executor/service/companion cleanup and transfer; main joins after Uvicorn returns. Launcher exit 42 or direct re-exec preserves planned-handoff environment. Cleanup may delay exit: no absolute bound. Unexpected cleanup/transfer errors log unconfirmed custody and exit 1. Restart skips port sweeps: exit/exec closes its listeners; a port is no ownership proof. Ordinary close closes Host Service; launcher/recovery/Panic sweeps remain separate, prove no PID identity and may stop unrelated listeners.
|
||||
Teardown sets the stop flag, joins the supervisor within its bound and terminalizes `kill_workers` before optional waits/sweeps. SIGKILL may lose writes/late frames; snapshot restore recovers missed writes. Queue init hands over `state/direct_roots.json` before clearing it, fencing listed roots like surviving RUNNING rows as Cancelled, not infra-failed. The restart watcher owns final worker/executor/service/companion cleanup and transfer; main joins after Uvicorn returns. Launcher exit 42 or the direct transfer preserves planned-handoff environment: POSIX re-execs in place; Windows, whose C runtime only emulates exec (unquoted arguments, a fragile environment block), spawns the custodied successor and exits 42. The token is no recovery acknowledgement there: direct recovery proof still requires the same PID. Cleanup may delay exit: no absolute bound. Unexpected cleanup/transfer errors log unconfirmed custody and exit 1. Restart skips port sweeps: exit/exec closes its listeners; a port is no ownership proof. Ordinary close closes Host Service; launcher/recovery/Panic sweeps remain separate, prove no PID identity and may stop unrelated listeners.
|
||||
|
||||
Manual Restart uses `server_restart._perform_owner_restart`, re-exported by `server`, with transport notices. HTTP/WS buttons use this owner until supervisor/bridge readiness. Typed WS chat retains its canonical row/echo; Host Service retains correlation and checks the known owner pair. Ready dispatch keeps its envelope; Panic keeps its early bypass. Before stopping anything, `_safe_restart_serialized` locks updates, checks the managed-update transaction and runs `safe_restart` checkout/dependency/import/stable fallback. Refusal, including an unwritable no-resume flag, reports “Restart cancelled” and leaves the server intact.
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ The one dependency authority and its packaging projections are ARCHITECTURE §8
|
|||
|
||||
Default local pytest excludes seven costly or environment-dependent lanes — `integration`, `browser`, `ui_browser`, `ui_browser_docker`, `portable_detail`, `skill_smoke` and `size_ratchet` — and CI opts into them explicitly (job topology and provider matrix: ARCHITECTURE §8 "CI topology"):
|
||||
|
||||
- `integration` runs real provider checks, including the trusted direct-OpenAI canary rows derived from `OPENAI_DIRECT_DEFAULTS`. Missing core credentials are red in the official repository job; quota/429/5xx/timeout may be typed inconclusive, while contract/auth/model/reasoning/tool 4xx stay red; on an official-repository release tag the job's last step (`tests/provider_release_floor.py`) also fails unless every required canary passed, and elsewhere it only reports. Secretless request-wire and Anthropic-custody contracts remain in ordinary pull-request tests: do not move provider secrets into PR jobs or duplicate the trusted lane. The KEYLESS `tests/system_e2e/` scenario lane rides the same marker plus `serial` and the `OUROBOROS_E2E_DEEP=mock` env gate (real isolated servers, no provider keys: ARCHITECTURE §8 "System E2E suite"); those three gates shut it out of every other pass, so only the dedicated `system-e2e-mock` job runs it — daily schedule, manual dispatch or release tag, never an ordinary push or pull request, carrying no secret.
|
||||
- `integration` runs real provider checks, including the trusted direct-OpenAI canary rows derived from `OPENAI_DIRECT_DEFAULTS`. Missing core credentials are red in the official repository job; quota/429/5xx/timeout may be typed inconclusive, while contract/auth/model/reasoning/tool 4xx stay red; on an official-repository release tag the job's last step (`tests/provider_release_floor.py`) also fails unless every required canary passed, and elsewhere it only reports. Secretless request-wire and Anthropic-custody contracts remain in ordinary pull-request tests: do not move provider secrets into PR jobs or duplicate the trusted lane. The KEYLESS `tests/system_e2e/` scenario lane rides the same marker plus `serial` and the `OUROBOROS_E2E_DEEP=mock` env gate (real isolated servers, no provider keys: ARCHITECTURE §8 "System E2E suite"); those three gates shut it out of every other pass, so only the dedicated `system-e2e-mock` job runs it — manual dispatch or release tag, never an ordinary push or pull request, carrying no secret.
|
||||
- `browser` / `ui_browser` / `ui_browser_docker` launch real Playwright engines (agent browser tools / the host UI / the `ouroboros-web:test` container; the docker lane skips cleanly when Docker is unavailable locally). The marker is the source of truth for what the lane collects; the four Widgets lifecycle suites listed under "Declarative widgets" run in it. `portable_detail` covers build/portable artifact invariants. `ui-smoke` runs the complete existing `ui_browser` lane on non-documentation PRs and on EVERY push to `ouroboros`; manual/tag runs also retain Chromium/WebKit browser-tool coverage (triggers: ARCHITECTURE §8; required invocation and its guards: "Safe local verification" below). None of these joins the default local run or needs a paid provider.
|
||||
- `skill_smoke` installs the nine pinned official skills from the LIVE catalog (list in `tests/test_skill_smoke_official.py`) as the dedicated 3-OS CI job, in serial pytest invocations with real network and real pip; red means investigate — there is deliberately no fallback-skip. Its paid review tier runs as a SEPARATE pytest step, ORDERED FIRST and ubuntu-only, alone carrying the provider key (why: ARCHITECTURE §8 "CI topology"); a missing key is a hard red, not a skip.
|
||||
- `size_ratchet` carries the live-repo size gates and is the ONLY blocking surface for repository size (rules under "Module Size & Complexity"; base fallback and the fail-closed rule: ARCHITECTURE §8 "CI topology"); only checks against the live repo carry the marker.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import asyncio
|
|||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
|
@ -383,6 +383,7 @@ def _annotate_terminal_task_truth(
|
|||
historical_terminals: Optional[dict] = None,
|
||||
deferred_lineage: Optional[set] = None,
|
||||
evidence_anchors: Optional[dict] = None,
|
||||
evidence_scope: Optional[Callable[[str], bool]] = None,
|
||||
) -> None:
|
||||
"""Project task truth AFTER quotas, paying only for represented/current tasks.
|
||||
|
||||
|
|
@ -410,7 +411,10 @@ def _annotate_terminal_task_truth(
|
|||
task_id = str(message.get("task_id") or "")
|
||||
if not task_id or message.get("system_type") == "project_question_pointer":
|
||||
continue
|
||||
represented.setdefault(task_id, str(message.get("ts") or ""))
|
||||
# A room may show a row its task does not live in (Main pins a Project's
|
||||
# notice and lifecycle rows); tool evidence follows the task's own route.
|
||||
if evidence_scope is None or evidence_scope(task_id):
|
||||
represented.setdefault(task_id, str(message.get("ts") or ""))
|
||||
if type(message.get("tool_calls")) is int:
|
||||
tool_counts[task_id] = max(tool_counts.get(task_id, 0), message["tool_calls"])
|
||||
progress, summary = bool(message.get("is_progress")), message.get("system_type") == "task_summary"
|
||||
|
|
@ -1481,6 +1485,8 @@ def _assemble_history_response(
|
|||
historical_terminals=historical_terminals,
|
||||
deferred_lineage=deferred_lineage,
|
||||
evidence_anchors=evidence_anchors,
|
||||
evidence_scope=lambda task_id: task_id not in bindings_by_task or row_matches_thread.evidence_matches(
|
||||
bindings_by_task[task_id], {"task_id": task_id}),
|
||||
)
|
||||
for source in ("chat", "progress"):
|
||||
before[source] = max([before[source], *(entry["_history_end"] for entry in selections[source][0] or ()
|
||||
|
|
|
|||
|
|
@ -74,10 +74,11 @@ def adopt_session_id(value: str) -> None:
|
|||
next reap tick. The worker entrypoint calls this with the server's id.
|
||||
|
||||
The id is passed as a spawn ARGUMENT, never via ambient env: an env var
|
||||
would survive ``server_control.restart_current_process`` (which re-execs
|
||||
with ``os.environ.copy()``), making a freshly restarted server adopt the
|
||||
dead generation's id and treat leftover processes as same-session
|
||||
survivors — the inverse leak. A spawn arg cannot survive an exec.
|
||||
would survive ``server_control.restart_current_process`` (which hands the
|
||||
server over with ``os.environ.copy()``: exec on POSIX, spawn on Windows),
|
||||
making a freshly restarted server adopt the dead generation's id and treat
|
||||
leftover processes as same-session survivors — the inverse leak. A spawn
|
||||
arg survives neither transfer.
|
||||
"""
|
||||
global _SESSION_ID
|
||||
v = str(value or "").strip()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import pathlib
|
|||
import sys
|
||||
from typing import Any
|
||||
|
||||
from ouroboros.platform_layer import IS_WINDOWS
|
||||
|
||||
|
||||
def external_owner_binding(state: dict) -> tuple[bool, Any, tuple[int, int]]:
|
||||
"""Ordinary commands use this current state observation, never Panic's cached pair."""
|
||||
|
|
@ -105,7 +107,7 @@ def restart_current_process(
|
|||
log: Any,
|
||||
owner_initiated: bool = False,
|
||||
) -> None:
|
||||
"""Re-exec this server process.
|
||||
"""Hand this server over to its successor: exec on POSIX, supervised spawn on Windows.
|
||||
|
||||
``owner_initiated`` marks the restart the OWNER asked for (the chat Restart
|
||||
button, and the control endpoints that restart on the owner's behalf). Only
|
||||
|
|
@ -148,33 +150,46 @@ def restart_current_process(
|
|||
except Exception:
|
||||
raw_argv = sys.argv
|
||||
argv = [sys.executable, *raw_argv]
|
||||
log.info("Re-executing direct server mode on %s:%d", desired_host, port)
|
||||
try:
|
||||
os.execvpe(sys.executable, argv, env)
|
||||
except Exception:
|
||||
log.exception("Direct re-exec failed; attempting spawned restart fallback.")
|
||||
if IS_WINDOWS:
|
||||
# Windows has no process-image replacement. Its C runtime emulates exec by
|
||||
# starting an UNQUOTED command line and exiting the caller, so a spaced or
|
||||
# empty argument (a Python under "Program Files") reaches the successor
|
||||
# broken. The supervised spawn below quotes argv and IS the transfer here.
|
||||
log.info("Starting the replacement direct server on %s:%d", desired_host, port)
|
||||
else:
|
||||
log.info("Re-executing direct server mode on %s:%d", desired_host, port)
|
||||
try:
|
||||
from ouroboros.config import DATA_DIR
|
||||
from ouroboros.process_custody import spawn_supervised
|
||||
|
||||
spawn_supervised(
|
||||
argv,
|
||||
drive_root=pathlib.Path(DATA_DIR),
|
||||
# daemon, NOT session: the replacement IS the next server
|
||||
# generation. A session-scoped entry carries this dying
|
||||
# generation's session id, so the new server's startup reap
|
||||
# would see it as a foreign-session process and SIGKILL itself.
|
||||
# daemon scope is always a reaper survivor (launcher-managed
|
||||
# lifecycle), which is correct for a long-lived top-level server.
|
||||
purpose="server_restart_fallback",
|
||||
scope="daemon",
|
||||
cwd=str(repo_dir),
|
||||
env=env,
|
||||
)
|
||||
log.info("Spawned replacement server process after exec failure.")
|
||||
os.execvpe(sys.executable, argv, env)
|
||||
return # exec does not return; only a test double of it reaches this line
|
||||
except Exception:
|
||||
log.exception("Spawned restart fallback failed; no successor was started.")
|
||||
raise
|
||||
log.exception("Direct re-exec failed; attempting spawned restart fallback.")
|
||||
try:
|
||||
from ouroboros.config import DATA_DIR
|
||||
from ouroboros.process_custody import spawn_supervised
|
||||
|
||||
spawn_supervised(
|
||||
argv,
|
||||
drive_root=pathlib.Path(DATA_DIR),
|
||||
# daemon, NOT session: the replacement IS the next server
|
||||
# generation. A session-scoped entry carries this dying
|
||||
# generation's session id, so the new server's startup reap
|
||||
# would see it as a foreign-session process and SIGKILL itself.
|
||||
# daemon scope is always a reaper survivor (launcher-managed
|
||||
# lifecycle), which is correct for a long-lived top-level server.
|
||||
purpose="server_restart_fallback",
|
||||
scope="daemon",
|
||||
# Windows: the successor keeps the caller's console group, as exec
|
||||
# would (a new group ignores CTRL+C). POSIX fallback: a new session,
|
||||
# so a failed custody write kills only the successor and the
|
||||
# caller's terminal hangup does not reach it.
|
||||
new_process_group=not IS_WINDOWS,
|
||||
cwd=str(repo_dir),
|
||||
env=env,
|
||||
)
|
||||
log.info("Spawned the replacement server process.")
|
||||
except Exception:
|
||||
log.exception("Spawned restart fallback failed; no successor was started.")
|
||||
raise
|
||||
|
||||
|
||||
def execute_panic_stop(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
127
|
||||
The task_evidence carrier stays in the task's own room: a row Main pins for a Project task mints no carrier or card in Main.
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
269
|
||||
The direct restart's transfer per platform: POSIX re-execs in place, Windows spawns the custodied successor because its C runtime only emulates exec; the token is no recovery acknowledgement there.
|
||||
|
|
@ -92,6 +92,8 @@ def direct_exec_environment(monkeypatch, root):
|
|||
captured.update(env)
|
||||
raise ReachedExec()
|
||||
monkeypatch.setattr(control.os, "execvpe", execvpe)
|
||||
monkeypatch.setattr("ouroboros.process_custody.spawn_supervised",
|
||||
lambda argv, **kw: execvpe(argv[0], argv, kw["env"]))
|
||||
with pytest.raises(ReachedExec):
|
||||
server._restart_current_process("127.0.0.1", 8765)
|
||||
return captured
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ def test_one_browser_lane_serves_pull_requests_manual_tags_and_ouroboros_pushes(
|
|||
# A push reaches the lane through its own workflow, never through ci.yml's
|
||||
# push trigger.
|
||||
assert "push" not in caller["if"] and "schedule" not in caller["if"]
|
||||
assert ci_triggers["schedule"] == [{"cron": "37 4 * * *"}]
|
||||
assert "schedule" not in ci_triggers # owner, 2026-10-05: no nightly run nobody reads
|
||||
|
||||
assert list(push_triggers) == ["push", "workflow_dispatch"]
|
||||
assert push_triggers["workflow_dispatch"]["inputs"]["diagnostic"]["options"] == [
|
||||
|
|
|
|||
|
|
@ -516,7 +516,7 @@ def test_delivery_binds_published_bytes_to_submitted_digest(tmp_path, monkeypatc
|
|||
}
|
||||
config, executor = _stub_terminal_task_executor(tmp_path, monkeypatch, gateway_result)
|
||||
original, updated = b"synthetic-original-marker", b"synthetic-later-marker"
|
||||
markers, submitted = [], []
|
||||
markers, submitted, argvs = [], [], []
|
||||
calls = {"gateway": 0, "query": 0}
|
||||
|
||||
def generate(_task, workspace, _agent_id):
|
||||
|
|
@ -530,10 +530,9 @@ def test_delivery_binds_published_bytes_to_submitted_digest(tmp_path, monkeypatc
|
|||
return dict(gateway_result)
|
||||
|
||||
def command(argv, **_kwargs):
|
||||
assert argv[:1] == ["docker"] and "exec" in argv
|
||||
# This injected command observes the host-shaped argv; the benchmark
|
||||
# itself uses POSIX mounts, while this byte-custody test runs on Windows too.
|
||||
assert str(pathlib.Path("/workspace/final.poc").resolve()) in argv
|
||||
# Record, do not assert, here: after the paid gateway call the executor
|
||||
# turns any exception into an infra_failed row, hiding the cause.
|
||||
argvs.append(list(argv))
|
||||
submitted.append(markers[0].read_bytes())
|
||||
if change == "changed_back_after_submit":
|
||||
markers[0].write_bytes(original)
|
||||
|
|
@ -561,6 +560,10 @@ def test_delivery_binds_published_bytes_to_submitted_digest(tmp_path, monkeypatc
|
|||
monkeypatch.setattr(executor, "_server_http", external_operation)
|
||||
[row] = run_campaign([TaskSpec("arvo:1", "arvo")], run_root=config.run_root,
|
||||
executor=executor.run_task, estimated_cost_usd=1, budget_cap_usd=10)
|
||||
# Only --host is host-shaped; the rest runs inside the Linux container and
|
||||
# stays a literal on every host (a resolved D:\workspace\final.poc names no file).
|
||||
assert argvs == [["docker", "--host", executor.host.value, "exec", "--workdir", "/workspace",
|
||||
"workspace-123", "bash", "/workspace/submit.sh", "/workspace/final.poc"]]
|
||||
persisted = [json.loads(line) for line in (config.run_root / "result_index.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert persisted == [row]
|
||||
assert calls["gateway"] == len(submitted) == 1
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ secret.
|
|||
|
||||
Pinned as a contract, not as text: the job fires only on a dispatch whose
|
||||
`e2e_live` input is true (never a plain dispatch — the pre-tag 3-OS matrix must
|
||||
not spend money — nor push, pull_request, tag, or any schedule: the workflow's
|
||||
one cron belongs to the keyless lane); the input changes no other job's gate;
|
||||
not spend money — nor push, pull_request, tag, or any schedule: the workflow
|
||||
carries none); the input changes no other job's gate;
|
||||
the dispatch checks out and seeds its own sha; it names
|
||||
exactly one secret, `OUROBOROS_E2E_LIVE_OPENROUTER_KEY`, gated through a
|
||||
non-secret job-level env (GitHub rejects `secrets.*` inside `if:`); a missing
|
||||
|
|
@ -40,7 +40,6 @@ REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|||
CI_PATH = REPO_ROOT / ".github" / "workflows" / "ci.yml"
|
||||
JOB = "e2e-live"
|
||||
SECRET = "OUROBOROS_E2E_LIVE_OPENROUTER_KEY"
|
||||
KEYLESS_CRON = "37 4 * * *"
|
||||
DISPATCH_INPUT = "e2e_live"
|
||||
SKIP_LINE = f"skipped: secret {SECRET} not configured"
|
||||
TOTAL_BUDGET_USD = 30.0
|
||||
|
|
@ -81,10 +80,9 @@ def _stand_args() -> dict[str, str | None]:
|
|||
def test_the_paid_lane_fires_only_on_an_opted_in_dispatch():
|
||||
workflow = _workflow()
|
||||
triggers = workflow.get("on") or workflow.get(True)
|
||||
# The schedule carries the keyless lane's cron alone: the paid stand runs on
|
||||
# no schedule (tests/test_platform_ci_events.py evaluates the gate per event).
|
||||
crons = [str(entry["cron"]) for entry in triggers["schedule"]]
|
||||
assert crons == [KEYLESS_CRON], crons
|
||||
# The workflow carries no schedule at all, and the gate refuses one anyway
|
||||
# (tests/test_platform_ci_events.py evaluates it per event).
|
||||
assert "schedule" not in triggers, triggers
|
||||
# The opt-in: a boolean dispatch input, OFF by default, naming the cost and
|
||||
# the secret. A plain `gh workflow run CI --ref <branch>` (the pre-tag 3-OS
|
||||
# matrix) therefore never runs the paid lane.
|
||||
|
|
|
|||
|
|
@ -333,3 +333,33 @@ def test_a_carrier_that_is_the_tasks_only_terminal_row_keeps_the_whole_terminal_
|
|||
assert carrier["task_terminal_status"] == "completed"
|
||||
for key in ("review_projection", "outcome_axes", "reason_code", "cancel_origin", "model_execution"):
|
||||
assert key in carrier and carrier[key] == summary[key], key
|
||||
|
||||
|
||||
@pytest.mark.parametrize("direction,row_type", [("out", "main_notice"), ("system", "project_handoff")])
|
||||
def test_a_row_main_pins_for_a_project_task_does_not_mint_its_card_in_main(tmp_path, direction, row_type):
|
||||
# Main shows a Project root's notice and lifecycle rows by a pinning exemption; the task
|
||||
# lives in its Project room, so its tool evidence (and the card it mints) stays there (#1505).
|
||||
from ouroboros.projects_registry import bind_task_to_project, create_project
|
||||
|
||||
project = create_project(tmp_path, "room", name="Project room")
|
||||
bind_task_to_project(tmp_path, "root", project["id"], origin={"absent": "system"})
|
||||
_write(tmp_path / "logs/chat.jsonl", [
|
||||
{"direction": direction, "type": row_type, "task_id": "root", "text": "Pinned", "ts": TS, "chat_id": 1},
|
||||
{"direction": "out", "task_id": "root", "text": "Project answer", "ts": TS, "chat_id": project["chat_id"]}])
|
||||
_write(tmp_path / "logs/tools.jsonl", [_tool("root")])
|
||||
_write(tmp_path / "task_results/root.json", [{"_schema_version": 1, "task_id": "root", "status": "completed",
|
||||
"chat_id": project["chat_id"]}])
|
||||
main = _room_read(tmp_path, 1)["messages"]
|
||||
assert [(row["task_id"], row["system_type"]) for row in main] == [("root", row_type)], main
|
||||
room = _room_read(tmp_path, project["chat_id"])["messages"]
|
||||
carriers = [row for row in room if row.get("system_type") == "task_evidence"]
|
||||
assert [row["task_id"] for row in carriers] == ["root"], "the Project room keeps its task's tools"
|
||||
assert carriers[0]["tool_evidence"]["observations"][0]["key"] == "tool:root:root-call"
|
||||
|
||||
|
||||
def test_a_main_rooted_tasks_own_main_notice_still_carries_its_tools(tmp_path):
|
||||
_write(tmp_path / "logs/chat.jsonl", [{"direction": "out", "type": "main_notice", "task_id": "root",
|
||||
"text": "Owner action", "ts": TS, "chat_id": 1}])
|
||||
_write(tmp_path / "logs/tools.jsonl", [_tool("root")])
|
||||
rows = _room_read(tmp_path, 1)["messages"]
|
||||
assert [(row["task_id"], row["system_type"]) for row in rows] == [("root", "main_notice"), ("root", "task_evidence")]
|
||||
|
|
|
|||
|
|
@ -985,7 +985,7 @@ def test_start_agent_never_overwrites_an_operator_env_host(monkeypatch, tmp_path
|
|||
|
||||
|
||||
def test_an_owner_restart_re_execs_without_the_inherited_runtime_mode_pin(monkeypatch, tmp_path):
|
||||
"""The re-exec env is the whole mechanism, so assert on the real env it hands execvpe.
|
||||
"""Assert on the environment passed to either platform's transfer primitive.
|
||||
|
||||
``OUROBOROS_BOOT_RUNTIME_MODE`` exists so a CHILD inherits the parent's ratchet
|
||||
baseline. Carried across the OWNER's own restart it also pinned the mode the
|
||||
|
|
@ -1004,6 +1004,8 @@ def test_an_owner_restart_re_execs_without_the_inherited_runtime_mode_pin(monkey
|
|||
captured["env"] = dict(env)
|
||||
|
||||
monkeypatch.setattr(os, "execvpe", _capture_exec)
|
||||
monkeypatch.setattr("ouroboros.process_custody.spawn_supervised",
|
||||
lambda argv, **kw: _capture_exec(argv[0], argv, kw["env"]))
|
||||
monkeypatch.setenv(BOOT_RUNTIME_MODE_ENV_KEY, "light")
|
||||
monkeypatch.setenv("OUROBOROS_RUNTIME_MODE", "advanced")
|
||||
|
||||
|
|
|
|||
|
|
@ -378,7 +378,7 @@ def test_requested_related_room_replay_keeps_one_physical_panel_per_cycle(harnes
|
|||
|
||||
class Executor(_HeldExecutor):
|
||||
def execute(self):
|
||||
self.execute_calls += 1
|
||||
super().execute() # held: the send settles only once the test releases it
|
||||
answer = (json.dumps([_finding("main", "need_evidence", locator="chat:1")])
|
||||
if self.execute_calls == 1 else CLEAN)
|
||||
return ReviewAttemptResult(message={"content": answer}, raw_text=answer,
|
||||
|
|
@ -386,13 +386,24 @@ def test_requested_related_room_replay_keeps_one_physical_panel_per_cycle(harnes
|
|||
|
||||
executor = Executor()
|
||||
monkeypatch.setattr("ouroboros.review_substrate._review_route_executor", lambda *a, **k: executor)
|
||||
|
||||
def dispatch():
|
||||
# A send that settles before the dispatch barrier registers the released
|
||||
# roster is returned inline and writes no mailbox frame; hold each send
|
||||
# until its dispatching call has returned at the barrier.
|
||||
executor.release.clear()
|
||||
try:
|
||||
return _call(ctx)
|
||||
finally:
|
||||
executor.release.set()
|
||||
|
||||
for cycle in (1, 2):
|
||||
_call(ctx)
|
||||
dispatch()
|
||||
wave = _state(harness)["waves"][-1]
|
||||
assert _wait_until(lambda: len(_mailbox_entries(harness.drive, ctx.task_id)) >= cycle)
|
||||
_collect(ctx, wave["request_fingerprint"])
|
||||
second = _state(harness)["waves"][-1]
|
||||
replay = _call(ctx)
|
||||
replay = dispatch()
|
||||
current = _state(harness)["waves"][-1]
|
||||
if current["request_fingerprint"] != second["request_fingerprint"]:
|
||||
assert _wait_until(lambda: len(_mailbox_entries(harness.drive, ctx.task_id)) >= 3)
|
||||
|
|
@ -402,7 +413,7 @@ def test_requested_related_room_replay_keeps_one_physical_panel_per_cycle(harnes
|
|||
assert executor.execute_calls == 2 and _state(harness)["cycles_paid"] == 2
|
||||
# A real update to requested evidence still earns the existing W3 refresh.
|
||||
append_jsonl(harness.drive / "logs/chat.jsonl", {"chat_id": 1, "text": "Main premise changed"})
|
||||
_call(ctx)
|
||||
dispatch()
|
||||
changed = _state(harness)["waves"][-1]
|
||||
assert _wait_until(lambda: len(_mailbox_entries(harness.drive, ctx.task_id)) >= 3)
|
||||
_collect(ctx, changed["request_fingerprint"])
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ def test_desktop_pr_matrix_keeps_merge_checkout_and_pr_base_evidence_secret_free
|
|||
assert _value(base, event="push", ref="refs/heads/ouroboros-stable") == "previous-tip"
|
||||
|
||||
|
||||
# "17 3 * * *" is a cron string this workflow does not carry: an event bearing
|
||||
# a stale cron still admits no ordinary job.
|
||||
# This workflow carries no schedule (owner, 2026-10-05): an event bearing any
|
||||
# cron, current or stale, admits no ordinary job.
|
||||
@pytest.mark.parametrize("cron", ["37 4 * * *", pytest.param("17 3 * * *", id="foreign-cron")])
|
||||
def test_scheduled_main_runs_do_not_enter_the_ordinary_matrix(cron):
|
||||
for name in ("quick-test", "full-test"):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from pathlib import Path
|
|||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
|
@ -137,23 +138,35 @@ def test_restart_fallback_runs_pin_handoff_before_retained_executor_cleanup(monk
|
|||
@pytest.mark.parametrize("uvicorn_returns", [False, True], ids=["held", "returned"])
|
||||
def test_direct_watchdog_physically_transfers_or_fails(tmp_path, failure, uvicorn_returns):
|
||||
"""Thread outcomes are physical exec or nonzero exit, never success/hang after an error."""
|
||||
script = tmp_path / "direct_reexec.py"
|
||||
receipt = tmp_path / "successor.json"
|
||||
script = tmp_path / "direct reexec.py"
|
||||
receipt = tmp_path / "successor receipt.json"
|
||||
script.write_text('''
|
||||
from contextlib import contextmanager
|
||||
import json, os, pathlib, sys, threading
|
||||
import json, os, pathlib, subprocess, sys, threading, time
|
||||
from types import SimpleNamespace
|
||||
|
||||
receipt = pathlib.Path(sys.argv[1])
|
||||
failure, uvicorn_returns = sys.argv[2], sys.argv[3] == "returned"
|
||||
if os.environ.get("OURO_TEST_SUCCESSOR"):
|
||||
from ouroboros.delegate_recovery import PLANNED_RESTART_TRANSACTION_ENV
|
||||
receipt.write_text(json.dumps({
|
||||
if os.getpid() != int(os.environ["OURO_TEST_SUCCESSOR"]):
|
||||
# A spawned successor reports only after the test saw its caller exit:
|
||||
# one that dies with the caller leaves no receipt.
|
||||
caller_exited = receipt.with_name(receipt.name + ".caller-exited")
|
||||
deadline = time.monotonic() + 20
|
||||
while not caller_exited.exists():
|
||||
if time.monotonic() > deadline:
|
||||
sys.exit(3)
|
||||
time.sleep(0.05)
|
||||
staged = receipt.with_name(receipt.name + ".tmp")
|
||||
staged.write_text(json.dumps({
|
||||
"pid": os.getpid(), "previous_pid": int(os.environ["OURO_TEST_SUCCESSOR"]),
|
||||
"transaction": os.environ.get(PLANNED_RESTART_TRANSACTION_ENV),
|
||||
"port": os.environ.get("OUROBOROS_SERVER_PORT"),
|
||||
"cleanup": os.environ.get("OURO_TEST_CLEANUP"),
|
||||
"argv": sys.argv[1:],
|
||||
}), encoding="utf-8")
|
||||
os.replace(staged, receipt) # the test polls for the receipt: never show it half-written
|
||||
sys.exit(0)
|
||||
|
||||
import server
|
||||
|
|
@ -199,18 +212,20 @@ server.write_port_file = lambda *args: None
|
|||
server.uvicorn.Config = lambda *args, **kwargs: None
|
||||
server._SignalStopServer = HeldServer
|
||||
server._emergency_process_cleanup = cleanup
|
||||
from ouroboros import server_control, process_custody
|
||||
if failure in {"reexec", "fallback"}:
|
||||
from ouroboros import server_control, process_custody
|
||||
server_control.os.execvpe = reexec_failure
|
||||
if failure == "reexec":
|
||||
process_custody.spawn_supervised = reexec_failure
|
||||
else:
|
||||
spawn = process_custody.spawn_supervised
|
||||
def joined_spawn(*args, **kwargs):
|
||||
child = spawn(*args, **kwargs)
|
||||
assert child.wait(timeout=10) == 0
|
||||
return child
|
||||
process_custody.spawn_supervised = joined_spawn
|
||||
if failure == "reexec":
|
||||
process_custody.spawn_supervised = reexec_failure
|
||||
elif os.name == "nt" or failure == "fallback":
|
||||
# Windows replaces by spawning. The successor gets its own stdio so the test
|
||||
# sees THIS process exit first; the successor then has to survive it to report.
|
||||
spawn = process_custody.spawn_supervised
|
||||
def detached_spawn(*args, **kwargs):
|
||||
log = open(receipt.with_name(receipt.name + ".successor.log"), "w", encoding="utf-8")
|
||||
kwargs.update(stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT)
|
||||
return spawn(*args, **kwargs)
|
||||
process_custody.spawn_supervised = detached_spawn
|
||||
supervisor.update_merge.read_update_tx_strict = lambda: ("absent", {})
|
||||
sys.exit(server.main())
|
||||
''', encoding="utf-8")
|
||||
|
|
@ -218,8 +233,9 @@ sys.exit(server.main())
|
|||
env.pop("OUROBOROS_SERVER_REEXEC_ARGV_JSON", None)
|
||||
env.pop("OURO_TEST_SUCCESSOR", None)
|
||||
env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1])
|
||||
result = subprocess.run([sys.executable, str(script), str(receipt), failure,
|
||||
"returned" if uvicorn_returns else "held"], env=env,
|
||||
arguments = [str(receipt), failure, "returned" if uvicorn_returns else "held",
|
||||
'argument with "quotes" and a trailing\\']
|
||||
result = subprocess.run([sys.executable, str(script), *arguments], env=env,
|
||||
capture_output=True, text=True, timeout=15)
|
||||
if failure in {"cleanup", "reexec"}:
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
|
|
@ -227,9 +243,81 @@ sys.exit(server.main())
|
|||
assert "fixture " + failure + " failure" in result.stderr
|
||||
assert not receipt.exists()
|
||||
return
|
||||
assert result.returncode == (42 if failure == "fallback" else 0), result.stdout + result.stderr
|
||||
spawned = os.name == "nt" or failure == "fallback"
|
||||
assert result.returncode == (42 if spawned else 0), result.stdout + result.stderr
|
||||
if spawned:
|
||||
receipt.with_name(receipt.name + ".caller-exited").write_text("", encoding="utf-8")
|
||||
deadline = time.monotonic() + 20
|
||||
while not receipt.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
successor_log = receipt.with_name(receipt.name + ".successor.log")
|
||||
assert receipt.exists(), "the spawned successor did not outlive its caller: " + (
|
||||
successor_log.read_text(encoding="utf-8") if successor_log.exists() else "no successor log")
|
||||
observed = json.loads(receipt.read_text(encoding="utf-8"))
|
||||
assert observed["transaction"] == "physical-handoff"
|
||||
assert observed["port"] == "9123" and observed["cleanup"] == "completed"
|
||||
if os.name != "nt":
|
||||
assert (observed["pid"] == observed["previous_pid"]) is (failure != "fallback")
|
||||
assert observed["argv"] == arguments
|
||||
assert (observed["pid"] == observed["previous_pid"]) is (not spawned)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("platform,failure", [
|
||||
("nt", ""), ("nt", "spawn"),
|
||||
("posix", ""), ("posix", "exec"), ("posix", "spawn"),
|
||||
])
|
||||
def test_direct_transfer_uses_platform_process_primitive(monkeypatch, tmp_path, platform, failure):
|
||||
"""Windows must bypass CRT exec; POSIX keeps exec and its spawn fallback."""
|
||||
from ouroboros import config, server_control, process_custody
|
||||
|
||||
class Transferred(BaseException):
|
||||
pass
|
||||
|
||||
calls, captured = [], {}
|
||||
argv = ["server path.py", "", 'argument with "quotes"', "trailing\\"]
|
||||
environment = {"OUROBOROS_SERVER_HOST": "127.0.0.1", "S1_TOKEN": "same generation",
|
||||
"OUROBOROS_MANAGED_BY_LAUNCHER": "1", "OUROBOROS_MANAGED_REPO_DIR": "old"}
|
||||
|
||||
def execvpe(executable, command, env):
|
||||
calls.append("exec")
|
||||
assert platform != "nt", "Windows restart must never enter CRT exec"
|
||||
captured.update(argv=command, env=env)
|
||||
if failure:
|
||||
raise OSError("exec failed")
|
||||
raise Transferred()
|
||||
|
||||
def spawn(command, **kwargs):
|
||||
calls.append("spawn")
|
||||
captured.update(argv=command, **kwargs)
|
||||
if failure == "spawn":
|
||||
raise OSError("spawn failed")
|
||||
return SimpleNamespace(pid=1234)
|
||||
|
||||
# Select the existing platform flag without changing pathlib's host OS.
|
||||
monkeypatch.setattr(server_control, "IS_WINDOWS", platform == "nt", raising=False)
|
||||
monkeypatch.setattr(server_control, "os", SimpleNamespace(
|
||||
name=platform, environ=environment, execvpe=execvpe))
|
||||
monkeypatch.setattr(server_control, "sys", SimpleNamespace(executable="python", argv=argv))
|
||||
monkeypatch.setattr(config, "load_settings", lambda: {})
|
||||
monkeypatch.setattr(config, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(process_custody, "spawn_supervised", spawn)
|
||||
logger = SimpleNamespace(info=lambda *a: None, exception=lambda *a: None)
|
||||
|
||||
def transfer():
|
||||
server_control.restart_current_process("127.0.0.1", 9123, repo_dir=tmp_path, log=logger)
|
||||
|
||||
if failure == "spawn":
|
||||
with pytest.raises(OSError, match="spawn failed"):
|
||||
transfer()
|
||||
elif platform == "posix" and not failure:
|
||||
with pytest.raises(Transferred):
|
||||
transfer()
|
||||
else:
|
||||
transfer()
|
||||
assert calls == (["spawn"] if platform == "nt" else ["exec"] + (["spawn"] if failure else []))
|
||||
assert captured["argv"] == ["python", *argv]
|
||||
assert captured["env"] == {"OUROBOROS_SERVER_HOST": "127.0.0.1",
|
||||
"S1_TOKEN": "same generation", "OUROBOROS_SERVER_PORT": "9123"}
|
||||
if "spawn" in calls:
|
||||
# Windows keeps the console group (CTRL+C); POSIX isolates the fallback's session.
|
||||
assert captured["new_process_group"] is (platform != "nt")
|
||||
assert captured["scope"] == "daemon"
|
||||
assert captured["drive_root"] == tmp_path and captured["cwd"] == str(tmp_path)
|
||||
|
|
|
|||
|
|
@ -134,6 +134,8 @@ def test_two_settings_restarts_do_not_invent_environment_authority(monkeypatch,
|
|||
monkeypatch.setattr(sys, "argv", argv)
|
||||
calls = []
|
||||
monkeypatch.setattr(os, "execvpe", lambda executable, argv, env: calls.append((executable, argv, env)))
|
||||
monkeypatch.setattr("ouroboros.process_custody.spawn_supervised",
|
||||
lambda argv, **kw: calls.append((argv[0], argv, kw["env"])))
|
||||
actual = "127.0.0.1"
|
||||
for desired in ("127.0.0.2", "127.0.0.4"):
|
||||
saved["OUROBOROS_SERVER_HOST"] = desired
|
||||
|
|
@ -154,6 +156,8 @@ def test_public_cli_reexec_argv_and_environment_override_remain_intact(monkeypat
|
|||
monkeypatch.setenv("OUROBOROS_SERVER_HOST", "0.0.0.0")
|
||||
captured = []
|
||||
monkeypatch.setattr(os, "execvpe", lambda executable, argv, env: captured.append((argv, env)))
|
||||
monkeypatch.setattr("ouroboros.process_custody.spawn_supervised",
|
||||
lambda argv, **kw: captured.append((argv, kw["env"])))
|
||||
restart_current_process("0.0.0.0", 8765, repo_dir=tmp_path, log=logging.getLogger("test"))
|
||||
assert captured[0][0] == [sys.executable, *argv]
|
||||
assert captured[0][1]["OUROBOROS_SERVER_HOST"] == "0.0.0.0"
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
"""The scheduled keyless `system-e2e-mock` CI job (owner 9A).
|
||||
"""The keyless `system-e2e-mock` CI job (owner 9A).
|
||||
|
||||
`tests/system_e2e/` is gated three ways — the `integration` and `serial`
|
||||
markers plus the `OUROBOROS_E2E_DEEP` env var — precisely so that no existing
|
||||
CI pytest pass can reach it. That is what makes a suite nobody executes: the
|
||||
gates work, and then nothing opens them. This job is the one thing that does,
|
||||
and the plan's §8 pull-request lane was replaced by a daily schedule (owner
|
||||
9A) because the scenarios spawn real isolated servers and cost minutes.
|
||||
on manual dispatch and on release tags; the plan's §8 pull-request lane gave
|
||||
way to a daily schedule (owner 9A), and the schedule itself was removed on
|
||||
2026-10-05 because nobody read the nightly results (owner).
|
||||
|
||||
Two properties are load-bearing enough to pin. The job must stay OFF push and
|
||||
pull_request, or the lane it was made cheap for becomes the slowest thing in
|
||||
every PR. And the daily schedule must not wake the PAID provider lane: a
|
||||
scheduled run carries the default branch in its ref, so `integration-test`
|
||||
leads with an explicit event guard that holds whatever ref conditions follow
|
||||
it, and no cron spends real provider credit.
|
||||
every PR. And no schedule may wake the PAID provider lane: a scheduled run
|
||||
carries the default branch in its ref, so `integration-test` leads with an
|
||||
explicit event guard that holds whatever ref conditions follow it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -91,42 +91,23 @@ def _job_text(job: str) -> str:
|
|||
return block.group(1)
|
||||
|
||||
|
||||
MOCK_CRON = "37 4 * * *"
|
||||
|
||||
|
||||
def test_the_workflow_carries_one_daily_off_peak_schedule_owned_by_one_job():
|
||||
def test_the_workflow_carries_no_schedule_and_no_job_admits_one():
|
||||
workflow = _workflow()
|
||||
schedule = _triggers(workflow).get("schedule") or []
|
||||
crons = [str(entry["cron"]) for entry in schedule]
|
||||
# One cron: this keyless lane. The paid `e2e-live` stand runs only on its
|
||||
# opt-in dispatch input (tests/test_e2e_live_ci_lane.py). A cron nobody
|
||||
# binds to is a second nightly wake-up of every job gated on the bare
|
||||
# event name.
|
||||
assert crons == [MOCK_CRON], schedule
|
||||
for entry in schedule:
|
||||
minute, hour, day, month, weekday = str(entry["cron"]).split()
|
||||
assert (day, month, weekday) == ("*", "*", "*"), entry
|
||||
assert minute.isdigit() and hour.isdigit(), "one fixed daily time, not a range"
|
||||
# On the hour is when everyone else's cron fires and GitHub's queue is
|
||||
# deepest; an off-peak minute is the documented way to avoid the backlog.
|
||||
assert int(minute) != 0, entry
|
||||
# Every job that fires on `schedule` names ITS cron string, so a cron added
|
||||
# for another lane never wakes it: a bare `github.event_name == 'schedule'` would.
|
||||
# Owner, 2026-10-05: nobody read the nightly results, so the workflow runs
|
||||
# nothing on a timer. A scheduled run would carry the default branch in its
|
||||
# ref; the remaining `!= 'schedule'` guards (integration-test) keep paid
|
||||
# lanes off one if a schedule ever returns.
|
||||
assert "schedule" not in _triggers(workflow), _triggers(workflow)
|
||||
for name, job in workflow["jobs"].items():
|
||||
condition = " ".join(str(job.get("if", "")).split())
|
||||
if "github.event_name == 'schedule'" not in condition:
|
||||
continue # `!= 'schedule'` guards (integration-test) keep a lane OFF every cron
|
||||
assert "github.event.schedule ==" in condition, (name, condition)
|
||||
assert "github.event_name == 'schedule' ||" not in condition, (name, condition)
|
||||
assert "github.event_name == 'schedule'" not in condition, (name, condition)
|
||||
|
||||
|
||||
def test_the_scheduled_lane_never_runs_on_a_push_or_a_pull_request():
|
||||
def test_the_lane_runs_only_on_a_dispatch_or_a_release_tag():
|
||||
job = _workflow()["jobs"][JOB]
|
||||
condition = " ".join(str(job["if"]).split())
|
||||
assert condition == (
|
||||
f"(github.event_name == 'schedule' && github.event.schedule == '{MOCK_CRON}')"
|
||||
" || github.event_name == 'workflow_dispatch'"
|
||||
" || startsWith(github.ref, 'refs/tags/v')"
|
||||
"github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v')"
|
||||
) # a release tag joins the lane to the release bar (batch №13 item 4); push/PR never, condition
|
||||
assert job["runs-on"] == "ubuntu-latest"
|
||||
# The budget must clear the suite, not merely exist. `> 0` accepted
|
||||
|
|
@ -138,7 +119,7 @@ def test_the_scheduled_lane_never_runs_on_a_push_or_a_pull_request():
|
|||
assert int(job["timeout-minutes"]) >= MIN_TIMEOUT_MINUTES
|
||||
|
||||
|
||||
def test_the_scheduled_lane_runs_the_keyless_suite_on_a_throwaway_root():
|
||||
def test_the_lane_runs_the_keyless_suite_on_a_throwaway_root():
|
||||
steps = _workflow()["jobs"][JOB]["steps"]
|
||||
assert [step.get("uses") for step in steps][:2] == [
|
||||
"actions/checkout@v4", "./.github/actions/setup-python-env",
|
||||
|
|
@ -161,7 +142,7 @@ def test_the_scheduled_lane_runs_the_keyless_suite_on_a_throwaway_root():
|
|||
assert all("runner.temp" in str(env[name]) for name in roots), env
|
||||
|
||||
|
||||
def test_the_scheduled_lane_uploads_its_servers_traces_and_never_a_settings_file():
|
||||
def test_the_lane_uploads_its_servers_traces_and_never_a_settings_file():
|
||||
"""Owner, 2026-10-04: the traces are the most useful part of a CI run. The
|
||||
scenario servers write their journals under pytest tmp_path trees — the
|
||||
OUROBOROS_* roots of the run steps are what tests/conftest.py isolates FROM —
|
||||
|
|
@ -201,14 +182,14 @@ def test_the_scheduled_lane_uploads_its_servers_traces_and_never_a_settings_file
|
|||
assert "TMPDIR" not in env and "OUROBOROS_TEST_TEMP_ROOT" not in env, step
|
||||
|
||||
|
||||
def test_the_scheduled_lane_asks_for_no_secret():
|
||||
def test_the_lane_asks_for_no_secret():
|
||||
"""Keyless by construction: a job gets a secret only by naming it."""
|
||||
assert "secrets." not in _job_text(JOB), _job_text(JOB)
|
||||
|
||||
|
||||
def test_the_daily_schedule_does_not_wake_the_paid_provider_lane():
|
||||
"""A scheduled run reports the default branch in github.ref. The leading
|
||||
event guard keeps the schedule off the paid lane whatever ref conditions
|
||||
def test_a_schedule_would_not_wake_the_paid_provider_lane():
|
||||
"""A scheduled run would report the default branch in github.ref. The leading
|
||||
event guard keeps any schedule off the paid lane whatever ref conditions
|
||||
follow it; the push workflow that serves branch pushes has no schedule."""
|
||||
condition = " ".join(str(_workflow()["jobs"]["integration-test"]["if"]).split())
|
||||
assert condition.startswith("github.event_name != 'schedule'"), condition
|
||||
|
|
|
|||
|
|
@ -226,8 +226,14 @@ def test_running_direct_turn_converts_to_project_and_its_answer_follows(
|
|||
page.wait_for_function("() => window.__testSockets?.[0]?.readyState === WebSocket.OPEN")
|
||||
page.wait_for_selector("#page-chat")
|
||||
page.get_by_text(marker, exact=False).first.wait_for(timeout=30000)
|
||||
# Zero convert buttons is also the state of a card whose
|
||||
# binding has not landed yet. Wait for the fact the
|
||||
# assertion below requires: no card, or a card that names
|
||||
# its project.
|
||||
reloaded = wait_until(
|
||||
lambda: (lambda facts: facts if facts["convert_buttons"] == 0 else None)(
|
||||
lambda: (lambda facts: facts if (
|
||||
not facts["card"] or facts["converted"] == "1" or facts["bound"] == "1"
|
||||
) else None)(
|
||||
page.evaluate(_MAIN_CARD_FACTS, task_id)),
|
||||
30) or page.evaluate(_MAIN_CARD_FACTS, task_id)
|
||||
assert reloaded["convert_buttons"] == 0, \
|
||||
|
|
|
|||
|
|
@ -153,18 +153,51 @@ def run_chat_viewport_smoke(
|
|||
# A scripted scroll (set_remaining) moves the view but decides nothing.
|
||||
deadline = time.monotonic() + 30
|
||||
evidence.checkpoint("read_to_latest:prepare")
|
||||
box = page.locator("#chat-messages").bounding_box()
|
||||
evidence.point = {"x": box["x"] + box["width"] / 2, "y": box["y"] + box["height"] / 2}
|
||||
page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
|
||||
|
||||
def feed_wheel_point():
|
||||
# .chat-live-card is overflow:hidden (web/style.css). WebKit treats
|
||||
# that as a scrollport and does not chain the wheel to #chat-messages.
|
||||
# The feed center is often that card. Cards are flex-start, so aim
|
||||
# at a point on the feed that is not inside such a scrollport.
|
||||
return page.locator("#chat-messages").evaluate(
|
||||
"""feed => {
|
||||
const box = feed.getBoundingClientRect();
|
||||
const y = box.top + box.height / 2;
|
||||
const blocksWheel = node => {
|
||||
for (let el = node; el && el !== feed; el = el.parentElement) {
|
||||
const overflow = getComputedStyle(el).overflowY;
|
||||
if (overflow === 'hidden' || overflow === 'auto' || overflow === 'scroll') return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
for (let x = box.right - 4; x > box.left + 4; x -= 8) {
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
if (hit && (hit === feed || feed.contains(hit)) && !blocksWheel(hit)) return {x, y};
|
||||
}
|
||||
return {x: box.left + box.width / 2, y};
|
||||
}"""
|
||||
)
|
||||
|
||||
# A native wheel impulse can finish short of the edge. Keep reading,
|
||||
# but never accept a settled gesture that made no progress toward it.
|
||||
# A wheel the engine swallows (seen, no scroll after two frames) is
|
||||
# not that gesture: retry it inside the same 30s deadline.
|
||||
while True:
|
||||
point = feed_wheel_point()
|
||||
evidence.point = point
|
||||
page.mouse.move(point["x"], point["y"])
|
||||
before = jump_state(page)
|
||||
page.evaluate("""() => {
|
||||
const feed = document.querySelector('#chat-messages');
|
||||
const state = window.__viewportWheel = {seen: false, scrolls: 0, settled: false};
|
||||
const wheel = () => { state.seen = true; };
|
||||
const scroll = () => { state.scrolls++; state.settled = false; };
|
||||
const state = window.__viewportWheel = {seen: false, scrolls: 0, settled: false, quiet: false};
|
||||
const wheel = () => {
|
||||
state.seen = true;
|
||||
const mark = state.scrolls;
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => {
|
||||
if (mark === state.scrolls) state.quiet = true;
|
||||
}));
|
||||
};
|
||||
const scroll = () => { state.scrolls++; state.settled = false; state.quiet = false; };
|
||||
const end = () => {
|
||||
if (!state.seen || !state.scrolls) return;
|
||||
const count = state.scrolls;
|
||||
|
|
@ -187,10 +220,21 @@ def run_chat_viewport_smoke(
|
|||
remaining_ms = (deadline - time.monotonic()) * 1000
|
||||
assert remaining_ms > 0, "reading did not reach the live edge within 30 seconds"
|
||||
page.wait_for_function(
|
||||
"atEdge => window.__viewportWheel.seen && (atEdge || window.__viewportWheel.settled)",
|
||||
arg=before["remaining"] <= 1, timeout=remaining_ms,
|
||||
"""() => {
|
||||
const state = window.__viewportWheel;
|
||||
const feed = document.querySelector('#chat-messages');
|
||||
if (!state || !state.seen) return false;
|
||||
const remaining = feed.scrollHeight - feed.scrollTop - feed.clientHeight;
|
||||
if (remaining <= 1 || state.settled) return true;
|
||||
return state.quiet && state.scrolls === 0;
|
||||
}""",
|
||||
timeout=remaining_ms,
|
||||
)
|
||||
after = jump_state(page)
|
||||
if after["remaining"] > 1 and page.evaluate(
|
||||
"() => { const s = window.__viewportWheel; return !s || s.scrolls === 0; }"
|
||||
):
|
||||
continue
|
||||
assert before["remaining"] <= 1 or after["remaining"] < before["remaining"], (before, after)
|
||||
if after["remaining"] <= 1:
|
||||
break
|
||||
|
|
@ -816,6 +860,18 @@ def run_chat_viewport_smoke(
|
|||
'.chat-live-card[data-finished="0"]'
|
||||
).evaluate_all("cards => cards.map(card => card.dataset.taskId)")
|
||||
cancel_active_ids.append("vp-cancel-noop")
|
||||
set_remaining(page, 300)
|
||||
authority_anchor = visible_card_anchor(page)
|
||||
emit_frame(page, {**cancel_authority, "cancelable": True})
|
||||
assert page.locator(
|
||||
'.chat-live-card[data-task-id="vp-cancel-authority"] [data-cancel-run]'
|
||||
).count() == 1
|
||||
assert abs(card_top(page, authority_anchor["id"]) - authority_anchor["top"]) <= 6
|
||||
assert not jump_state(page)["dotHidden"]
|
||||
# A complete census that already lists this card mounts Stop
|
||||
# through syncCancelRunButton, which is not remote content.
|
||||
# The cancelable frame is then a duplicate and never sets the
|
||||
# activity bit. Install the census only after that bit is observed.
|
||||
page.route(
|
||||
"**/api/state",
|
||||
lambda route: route.fulfill(
|
||||
|
|
@ -828,14 +884,6 @@ def run_chat_viewport_smoke(
|
|||
} for task_id in cancel_active_ids]}),
|
||||
),
|
||||
)
|
||||
set_remaining(page, 300)
|
||||
authority_anchor = visible_card_anchor(page)
|
||||
emit_frame(page, {**cancel_authority, "cancelable": True})
|
||||
assert page.locator(
|
||||
'.chat-live-card[data-task-id="vp-cancel-authority"] [data-cancel-run]'
|
||||
).count() == 1
|
||||
assert abs(card_top(page, authority_anchor["id"]) - authority_anchor["top"]) <= 6
|
||||
assert not jump_state(page)["dotHidden"]
|
||||
button.click()
|
||||
page.evaluate(_SETTLE_TWO_FRAMES)
|
||||
page.wait_for_timeout(800) # drain the prior task_done history debounce
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue