mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-19 10:04:16 +00:00
fix: deliver owner controls while the supervisor initializes
Use the direct Panic or Restart owner until a live supervisor has published its command bridge. Preserve the ready consumer handoff and stale-bridge fallback, and exercise admission and deferred dispatch through HTTP. Move the existing manual restart operation unchanged into server_restart, retain the server re-export, and preserve the 1700-line composition bound.
This commit is contained in:
Ouroboros
parent
1e24d100c3
commit
f0773cb5d8
10 changed files with 154 additions and 83 deletions
|
|
@ -312,7 +312,7 @@ server.py (Starlette+uvicorn) ← HTTP + WebSocket on configurable host:port (de
|
|||
├── server_entrypoint.py ← CLI parsing + port binding helpers
|
||||
├── server_runtime.py ← Startup/onboarding wiring + WS liveness
|
||||
├── server_web.py ← `NoCacheStaticFiles`, web-dir resolver and fixed-source `read_author_kit_assets(repo_dir)` for optional author-owned routes; no endpoint or cache
|
||||
├── server_process.py, server_liveness.py, server_maintenance.py, server_restart.py, server_owner_routing.py, server_routing_context.py ← Server leaves the composition root calls: the facts one server process shares with every leaf; wedge detection for the supervisor generation; the upkeep a generation owes the drive; restart-adjacent helpers (shutdown, the owned-work stop of a manual Restart, the planned restart's engine-pin daemon stop; §9); where one owner message goes; the bounded facts one owner turn may address
|
||||
├── server_process.py, server_liveness.py, server_maintenance.py, server_restart.py, server_owner_routing.py, server_routing_context.py ← Server leaves the composition root calls: the facts one server process shares with every leaf; wedge detection for the supervisor generation; the upkeep a generation owes the drive; restart operations (shutdown, the checkout-first manual Restart, the planned restart's engine-pin daemon stop; §9); where one owner message goes; the bounded facts one owner turn may address
|
||||
├── task_continuation.py ← Durable review continuation state
|
||||
├── task_results.py ← Durable task results `task_results/<id>.json`; the locked `task_acceptance_review_accounting` claim (minted at first physical reviewer dispatch; a claim without a recoverable terminal host run is UNKNOWN, never permission to re-dispatch); the read-only root review-capacity projection is WALLET and cancellation only (`root_task_id`, `cap_cycles`, `claimed_cycles`, `remaining_cycles`, `binding_seen`, `dedupe`, `state`, `reason`), no time axis (§6 Task acceptance)
|
||||
├── task_result_schema.py ← Task-result schema admission: the `_schema_version` stamp, the classifier, and the quarantine an unstamped, future, malformed or retired-key row lands in
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ This chapter owns the ways this installation stops: ordinary window close, which
|
|||
|
||||
Ordinary window close preserves this installation's shared Claudexor daemon and work submitted by other clients, on graceful and forced launcher cleanup paths alike. It stops the server generation's workers and services, waits for its process group or Job, performs owned cleanup and releases the PID lock; the Windows daemon lives outside the launcher Job, and an old immutable launcher needs a later package update for this guarantee. A same-pin planned restart retains the daemon; a planned restart after a changed engine pin stops it in lifespan teardown (`server_restart._stop_owned_daemon_for_new_pin`: the landed `load_runtime_pin` compared with the serving engine through attach-only `read_owned_gateway`, then the explicit stop below), and an unpublished or unreadable pin or an unreachable, foreign or stopped daemon leaves the handoff unchanged. An unconfirmed stop retains custody and diagnostics while the restart proceeds; the next generation reconciles old runs without inventing spend. In lifespan teardown the ONE irreversible durable write goes first — the stop flag, the bounded supervisor join, then `kill_workers`, which terminalizes every interrupted task — ahead of the best-effort extension-reconcile and host-service waits and the kill-all sweeps, because that order keeps the write reachable inside the launcher's force-exit budget (an external SIGKILL can still cut it short, and the later bridge and event-bus teardown can lose a late frame). Ordinary server shutdown closes its own Host Service listener; reserved-port sweeps remain separate launcher/recovery/Panic behaviour: they are not PID-identity proof and can terminate an unrelated listener on a reserved port.
|
||||
|
||||
The owner's manual Restart (`/restart`, the chat Restart button) is a clean stop of everything the current server generation owns, then the re-exec. The bridge delegates to `server._perform_owner_restart` with its transport notice callback. That common operation is checkout-first: `_safe_restart_serialized` (update lock, strict managed-update transaction gate, then `safe_restart`'s checkout, dependency sync, import test and stable fallback) runs before anything is stopped, so every refusal — an assisted-update merge mid-resolution, a failed checkout, an unwritable no-resume flag — leaves the server intact and is answered with one "Restart cancelled" line. Past that point the restart always follows: the durable no-resume intent (`state/owner_restart_no_resume.flag` plus the `panic_stop.flag` compatibility pair, consumed by `auto_resume_after_restart`), then `server_restart._stop_owned_work` — one immediate cancel intent per RUNNING task, live direct activity and in-flight post-task synthesis (`cancel_intents.request_cancel`, source `owner_restart`), `kill_workers`, delegated-run cancellation through the owner-gone seam `reconcile_orphaned_runs(running_task_ids=set())` over `read_owned_gateway`, and the attested owned-daemon `stop_outcome()` exactly as Panic makes it. Everything between the cancel intents and the daemon stop is attach-only: an `ensure_owned_gateway` there would start a dead daemon. Nothing is a veto or a deferral: an unconfirmed or raising worker shutdown leaves a critical log, an unconfirmed or raising daemon stop the `process_stop_unconfirmed` supervisor row; custody stays retained and the restart proceeds. The next generation does its ordinary startup work: owner restart is a no-resume cause (`delegate_recovery.NO_RESUME_CAUSES`), so nothing is adopted; the startup custody sweep reconciles every open delegated run as owner-gone (a run the stopped daemon took down answers absent → `close_absent_run`, no invented spend; a journal-recovered terminal settles; a pending invocation replays under its own key), and an in-flight owned model operation stays a `dispatched`/`unresolved` physical-attempt row that already counts as spend upper bound. After an unconfirmed stop the new generation attaches to the still-live daemon rather than spawning a second one, and the lifespan's one background `warm_owned_daemon()` (provisioned homes only) makes the first delegation after a Restart find the daemon serving. The explicit stop is installation-wide — manual Restart ends runs served by the owned daemon, another client's included — which differs intentionally from ordinary window close; planned self-restart, managed-update handoff and Panic keep their separate contracts.
|
||||
The owner's manual Restart (`/restart`, the chat Restart button) is a clean stop of everything the current server generation owns, then the re-exec. The bridge delegates to `server_restart._perform_owner_restart` (re-exported by `server`) with its transport notice callback. Before a supervisor publishes its bridge, HTTP admission binds these same Restart and Panic operations directly; deferred execution hands off only to a live supervisor with a published bridge. A thread still initializing, or a stale bridge left by a dead thread, cannot consume a command, so the direct owner executes it. That common operation is checkout-first: `_safe_restart_serialized` (update lock, strict managed-update transaction gate, then `safe_restart`'s checkout, dependency sync, import test and stable fallback) runs before anything is stopped, so every refusal — an assisted-update merge mid-resolution, a failed checkout, an unwritable no-resume flag — leaves the server intact and is answered with one "Restart cancelled" line. Past that point the restart always follows: the durable no-resume intent (`state/owner_restart_no_resume.flag` plus the `panic_stop.flag` compatibility pair, consumed by `auto_resume_after_restart`), then `server_restart._stop_owned_work` — one immediate cancel intent per RUNNING task, live direct activity and in-flight post-task synthesis (`cancel_intents.request_cancel`, source `owner_restart`), `kill_workers`, delegated-run cancellation through the owner-gone seam `reconcile_orphaned_runs(running_task_ids=set())` over `read_owned_gateway`, and the attested owned-daemon `stop_outcome()` exactly as Panic makes it. Everything between the cancel intents and the daemon stop is attach-only: an `ensure_owned_gateway` there would start a dead daemon. Nothing is a veto or a deferral: an unconfirmed or raising worker shutdown leaves a critical log, an unconfirmed or raising daemon stop the `process_stop_unconfirmed` supervisor row; custody stays retained and the restart proceeds. The next generation does its ordinary startup work: owner restart is a no-resume cause (`delegate_recovery.NO_RESUME_CAUSES`), so nothing is adopted; the startup custody sweep reconciles every open delegated run as owner-gone (a run the stopped daemon took down answers absent → `close_absent_run`, no invented spend; a journal-recovered terminal settles; a pending invocation replays under its own key), and an in-flight owned model operation stays a `dispatched`/`unresolved` physical-attempt row that already counts as spend upper bound. After an unconfirmed stop the new generation attaches to the still-live daemon rather than spawning a second one, and the lifespan's one background `warm_owned_daemon()` (provisioned homes only) makes the first delegation after a Restart find the daemon serving. The explicit stop is installation-wide — manual Restart ends runs served by the owned daemon, another client's included — which differs intentionally from ordinary window close; planned self-restart, managed-update handoff and Panic keep their separate contracts.
|
||||
|
||||
Panic is a complete explicit owner stop, not a restart: stop consciousness, record evolution owner-stop state, close the campaign and pending promotion, write `panic_stop.flag`, stop local models and call `OwnedClaudexorDaemon.stop_outcome` before worker cleanup, then end tracked foreground commands, executor processes, services, companions and workers before hard server exit. The shared daemon may belong to a previous generation and is outside the Windows launcher Job, so its explicit stop, not closing that Job, owns termination; a failure emits a critical `process_stop_unconfirmed` row and retains unresolved custody while the rest of Panic continues. The next launch suppresses automatic work under the panic/no-resume flags.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Machine extraction of the `docs/ARCHITECTURE.md` "Data layout (`~/Ouroboros/`)" tree — the durable-file orientation carrier (this tree's counterpart of the reference PERSISTENCE_OWNERS derivation checklist) — regenerated by `python scripts/regenerate_inventories.py`. Do not edit. Every entry is probed against reality: repo entries must exist as tracked paths; data-plane entries must appear as a literal in the runtime sources that construct them. A durable file renamed or removed in code while its tree row survives = red (`tests/test_generated_inventories.py`).
|
||||
|
||||
Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 578-667; UTF-8 SHA-256 `1ae3658e81385efd0b279daeb7203819eda4d2058e3c8eccbb8783d73d6ab23f`.
|
||||
Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 578-667; UTF-8 SHA-256 `478f45bc459a8464852fc53c9d10671146adae1267ed600947fd80408ab35757`.
|
||||
|
||||
- entries: **78** (code-ref: 71, repo-dir: 6, repo-path: 1)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
AST-derived inventory of compatibility facades, regenerated by `python scripts/regenerate_inventories.py`. Do not edit. A facade row is any runtime module whose top-level `from <population module> import ...` statements carry the `noqa: F401` re-export marker — the codebase's declared "this binding exists for its binding, not for this module's own use" convention (reference FACADE_CONSUMERS method). Leaf domains come from `ouroboros/domains.toml`; a leaf outside the facade's domain is marked ✗ (that edge also appears in the manifest's pinned direction matrix). `tests/test_generated_inventories.py` pins byte-identity, so any re-export surface change must regenerate this file.
|
||||
|
||||
- facade modules: **58**; marked re-export bindings: **2358**; cross-domain facade→leaf pairs: **131**
|
||||
- facade modules: **58**; marked re-export bindings: **2359**; cross-domain facade→leaf pairs: **131**
|
||||
|
||||
| facade | domain | bindings | leaves |
|
||||
|---|---|---:|---|
|
||||
|
|
@ -55,7 +55,7 @@ AST-derived inventory of compatibility facades, regenerated by `python scripts/r
|
|||
| `ouroboros/tools/shell_process.py` | D05 | 1 | `ouroboros/tools/process_facts.py` (1 ✗D04) |
|
||||
| `ouroboros/tools/subagent_integration.py` | D07 | 13 | `ouroboros/headless.py` (2 ✗D17)<br>`ouroboros/tools/subagent_integration_delegated.py` (11) |
|
||||
| `ouroboros/usage_accounting.py` | D16 | 42 | `ouroboros/_usage_cache_splits.py` (4)<br>`ouroboros/_usage_rows.py` (6)<br>`ouroboros/_usage_rows_memo.py` (6)<br>`ouroboros/usage_ledger.py` (18)<br>`ouroboros/usage_legacy_import.py` (5)<br>`ouroboros/utils.py` (3 ✗D18) |
|
||||
| `server.py` | D11 | 50 | `ouroboros/server_liveness.py` (4)<br>`ouroboros/server_maintenance.py` (14)<br>`ouroboros/server_owner_routing.py` (5)<br>`ouroboros/server_process.py` (6)<br>`ouroboros/server_restart.py` (7)<br>`ouroboros/server_routing_context.py` (14) |
|
||||
| `server.py` | D11 | 51 | `ouroboros/server_liveness.py` (4)<br>`ouroboros/server_maintenance.py` (14)<br>`ouroboros/server_owner_routing.py` (5)<br>`ouroboros/server_process.py` (6)<br>`ouroboros/server_restart.py` (8)<br>`ouroboros/server_routing_context.py` (14) |
|
||||
| `supervisor/events.py` | D08 | 94 | `ouroboros/config.py` (1 ✗D12)<br>`ouroboros/contracts/task_constraint.py` (1 ✗D19)<br>`ouroboros/cost_projection.py` (3 ✗D16)<br>`ouroboros/subagent_messages.py` (1 ✗D07)<br>`ouroboros/task_results.py` (2 ✗D17)<br>`ouroboros/tool_capabilities.py` (2 ✗D04)<br>`ouroboros/utils.py` (4 ✗D18)<br>`supervisor/cognitive_operations.py` (2)<br>`supervisor/events_budget.py` (4)<br>`supervisor/events_chat_delivery.py` (9)<br>`supervisor/events_coop_checkpoint.py` (6)<br>`supervisor/events_evolution_done.py` (1)<br>`supervisor/events_project_routing.py` (10)<br>`supervisor/events_runtime_controls.py` (6)<br>`supervisor/events_schedule_task.py` (4)<br>`supervisor/events_subagent_admission.py` (15)<br>`supervisor/events_task_done.py` (8)<br>`supervisor/events_worker_reports.py` (7)<br>`supervisor/log_addressing.py` (4)<br>`supervisor/queue_transitions.py` (1)<br>`supervisor/steering.py` (2 ✗D09)<br>`supervisor/task_dispatch.py` (1) |
|
||||
| `supervisor/git_ops.py` | D10 | 37 | `ouroboros/utils.py` (1 ✗D18)<br>`supervisor/git_ops_remotes.py` (4)<br>`supervisor/git_ops_rescue.py` (8)<br>`supervisor/git_ops_reset.py` (10)<br>`supervisor/git_ops_updates.py` (8)<br>`supervisor/state.py` (4 ✗D08)<br>`supervisor/update_recovery.py` (2) |
|
||||
| `supervisor/queue.py` | D08 | 102 | `ouroboros/config.py` (6 ✗D12)<br>`ouroboros/contracts/task_contract.py` (3 ✗D19)<br>`ouroboros/schedule_contract.py` (2)<br>`ouroboros/skill_loader.py` (1 ✗D14)<br>`ouroboros/utils.py` (3 ✗D18)<br>`supervisor/evolution_lifecycle.py` (12 ✗D15)<br>`supervisor/message_bus.py` (3)<br>`supervisor/queue_schedules.py` (12)<br>`supervisor/queue_snapshot.py` (5)<br>`supervisor/queue_timeouts.py` (8)<br>`supervisor/queue_transitions.py` (3)<br>`supervisor/schedule_time.py` (7)<br>`supervisor/state.py` (6)<br>`supervisor/task_admission.py` (9)<br>`supervisor/task_lifecycle.py` (17 ✗D09)<br>`supervisor/task_reaper.py` (5 ✗D09) |
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
|
||||
The live-task census the restart drain consults, the teardown arguments that
|
||||
finalize interrupted tasks with an honest reason, the managed-update guard on
|
||||
preserving queued work, the checkout/update serialization gate, the owned-work
|
||||
stop of the owner's manual Restart, the planned restart's engine-pin daemon stop,
|
||||
and the event bus shutdown live here. The restart transaction itself stays in
|
||||
preserving queued work, the checkout/update serialization gate, the owner's manual
|
||||
Restart operation, the planned restart's engine-pin daemon stop,
|
||||
and the event bus shutdown live here. The planned restart transaction itself stays in
|
||||
``server.py``: its planned-handoff transaction id joins the deferred drain record
|
||||
to the performer that raises the exit signal.
|
||||
"""
|
||||
|
|
@ -15,11 +15,54 @@ import pathlib
|
|||
import time
|
||||
from typing import Any
|
||||
|
||||
from ouroboros.server_process import DATA_DIR, _owner_restart_requested, _restart_requested, log
|
||||
from ouroboros.server_process import (
|
||||
DATA_DIR, _owner_restart_requested, _request_restart_exit, _restart_requested, log,
|
||||
)
|
||||
|
||||
_RESTARTABLE_UPDATE_PHASES = frozenset({"pending_boot_smoke", "applying_replace"})
|
||||
|
||||
|
||||
def _perform_owner_restart(ctx: Any, reply=None) -> tuple[bool, str]:
|
||||
"""Run the owner restart operation with an optional transport notice."""
|
||||
ok, restart_msg = _safe_restart_serialized(
|
||||
ctx.safe_restart,
|
||||
reason="owner_restart",
|
||||
unsynced_policy="rescue_and_reset",
|
||||
)
|
||||
if not ok:
|
||||
return False, restart_msg
|
||||
state_dir = DATA_DIR / "state"
|
||||
owner_restart_flag = state_dir / "owner_restart_no_resume.flag"
|
||||
stable_skip_flag = state_dir / "panic_stop.flag"
|
||||
try:
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
owner_restart_flag.write_text("owner_restart", encoding="utf-8")
|
||||
# Pair owner flag with panic_stop for stable-build auto-resume compatibility.
|
||||
stable_skip_flag.write_text("owner_restart_no_resume", encoding="utf-8")
|
||||
except Exception:
|
||||
owner_restart_flag.unlink(missing_ok=True)
|
||||
stable_skip_flag.unlink(missing_ok=True)
|
||||
log.warning("Failed to write owner restart no-resume flag", exc_info=True)
|
||||
return False, "could not write restart state."
|
||||
# Everything reversible is behind us (checkout landed, no-resume
|
||||
# intent durable): from here the restart always follows, and every
|
||||
# unconfirmed stop is a critical diagnostic, never a deferral.
|
||||
stopped_task_ids = _stop_owned_work(ctx)
|
||||
try:
|
||||
if reply is not None:
|
||||
# Say only what happened: with nothing owned the stop sentence
|
||||
# named a task that was never running.
|
||||
reply(
|
||||
"Stopping active task. New settings apply to the next message."
|
||||
if stopped_task_ids else "New settings apply to the next message.",
|
||||
"",
|
||||
)
|
||||
except Exception:
|
||||
log.warning("Failed to send owner restart stop notice; continuing restart", exc_info=True)
|
||||
_request_restart_exit(owner=True)
|
||||
return True, ""
|
||||
|
||||
|
||||
def _owned_live_task_ids(ctx: Any) -> list:
|
||||
"""Every id this generation's cancel intent can address: pooled tasks,
|
||||
in-process direct/ephemeral activities and running post-task synthesis."""
|
||||
|
|
|
|||
51
server.py
51
server.py
|
|
@ -99,6 +99,7 @@ from ouroboros.server_maintenance import ( # noqa: F401
|
|||
from ouroboros.server_restart import ( # noqa: F401
|
||||
_live_running_task_ids,
|
||||
_managed_update_pending_kwargs,
|
||||
_perform_owner_restart,
|
||||
_safe_restart_serialized,
|
||||
_shutdown_supervisor_event_bus,
|
||||
_shutdown_task_cleanup_args,
|
||||
|
|
@ -494,47 +495,6 @@ def _process_bridge_updates(bridge, offset: int, ctx: Any) -> int:
|
|||
return offset
|
||||
|
||||
|
||||
def _perform_owner_restart(ctx: Any, reply=None) -> tuple[bool, str]:
|
||||
"""Run the owner restart operation with an optional transport notice."""
|
||||
ok, restart_msg = _safe_restart_serialized(
|
||||
ctx.safe_restart,
|
||||
reason="owner_restart",
|
||||
unsynced_policy="rescue_and_reset",
|
||||
)
|
||||
if not ok:
|
||||
return False, restart_msg
|
||||
state_dir = DATA_DIR / "state"
|
||||
owner_restart_flag = state_dir / "owner_restart_no_resume.flag"
|
||||
stable_skip_flag = state_dir / "panic_stop.flag"
|
||||
try:
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
owner_restart_flag.write_text("owner_restart", encoding="utf-8")
|
||||
# Pair owner flag with panic_stop for stable-build auto-resume compatibility.
|
||||
stable_skip_flag.write_text("owner_restart_no_resume", encoding="utf-8")
|
||||
except Exception:
|
||||
owner_restart_flag.unlink(missing_ok=True)
|
||||
stable_skip_flag.unlink(missing_ok=True)
|
||||
log.warning("Failed to write owner restart no-resume flag", exc_info=True)
|
||||
return False, "could not write restart state."
|
||||
# Everything reversible is behind us (checkout landed, no-resume
|
||||
# intent durable): from here the restart always follows, and every
|
||||
# unconfirmed stop is a critical diagnostic, never a deferral.
|
||||
stopped_task_ids = _stop_owned_work(ctx)
|
||||
try:
|
||||
if reply is not None:
|
||||
# Say only what happened: with nothing owned the stop sentence
|
||||
# named a task that was never running.
|
||||
reply(
|
||||
"Stopping active task. New settings apply to the next message."
|
||||
if stopped_task_ids else "New settings apply to the next message.",
|
||||
"",
|
||||
)
|
||||
except Exception:
|
||||
log.warning("Failed to send owner restart stop notice; continuing restart", exc_info=True)
|
||||
_request_restart_exit(owner=True)
|
||||
return True, ""
|
||||
|
||||
|
||||
def _runtime_branch_defaults() -> tuple[str, str]:
|
||||
if _LAUNCHER_MANAGED:
|
||||
try:
|
||||
|
|
@ -1155,7 +1115,7 @@ def _execute_panic_stop(consciousness, kill_workers_fn) -> None:
|
|||
|
||||
def _startup_owner_command(command: str):
|
||||
"""Bind only process verbs while the normal command consumer is absent."""
|
||||
if command not in {"/panic", "/restart"} or (_supervisor_thread and _supervisor_thread.is_alive()):
|
||||
if command not in {"/panic", "/restart"}:
|
||||
return None
|
||||
|
||||
def execute():
|
||||
|
|
@ -1163,11 +1123,8 @@ def _startup_owner_command(command: str):
|
|||
# owns the ordinary command rather than two concurrent control paths.
|
||||
from supervisor.message_bus import try_get_bridge
|
||||
bridge = try_get_bridge()
|
||||
if _supervisor_thread and _supervisor_thread.is_alive():
|
||||
if bridge is not None:
|
||||
bridge.ui_send(command, broadcast=False)
|
||||
else:
|
||||
log.error("Startup owner command could not reach the initializing supervisor: %s", command)
|
||||
if _supervisor_thread and _supervisor_thread.is_alive() and bridge is not None:
|
||||
bridge.ui_send(command, broadcast=False)
|
||||
return
|
||||
from supervisor import state, workers, git_ops
|
||||
from types import SimpleNamespace
|
||||
|
|
|
|||
|
|
@ -89,8 +89,8 @@ def _restart(ctx, monkeypatch, *, gate=_gate):
|
|||
import server
|
||||
|
||||
exits = []
|
||||
monkeypatch.setattr(server, "_safe_restart_serialized", gate)
|
||||
monkeypatch.setattr(server, "_request_restart_exit", lambda owner=False: exits.append(owner))
|
||||
monkeypatch.setattr(server_restart, "_safe_restart_serialized", gate)
|
||||
monkeypatch.setattr(server_restart, "_request_restart_exit", lambda owner=False: exits.append(owner))
|
||||
server._process_bridge_updates(_bridge(), 0, ctx)
|
||||
return exits
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,8 @@ def startup_controls(tmp_path, monkeypatch):
|
|||
monkeypatch.setattr(server, '_consciousness', None)
|
||||
monkeypatch.setattr(server, '_restart_requested', threading.Event())
|
||||
monkeypatch.setattr(server, '_owner_restart_requested', threading.Event())
|
||||
from ouroboros import server_process
|
||||
from ouroboros import server_process, server_restart
|
||||
monkeypatch.setattr(server_restart, 'DATA_DIR', data)
|
||||
monkeypatch.setattr(server_process, '_restart_requested', server._restart_requested)
|
||||
monkeypatch.setattr(server_process, '_owner_restart_requested', server._owner_restart_requested)
|
||||
monkeypatch.setattr(message_bus, '_BRIDGE', None)
|
||||
|
|
@ -59,15 +60,52 @@ def startup_controls(tmp_path, monkeypatch):
|
|||
app = Starlette(routes=[Route('/api/command', api_command, methods=['POST'])])
|
||||
app.state.startup_owner_command = server._startup_owner_command
|
||||
with TestClient(app) as client:
|
||||
yield SimpleNamespace(server=server, client=client, data=data, repo=repo)
|
||||
yield SimpleNamespace(server=server, client=client, app=app, data=data, repo=repo)
|
||||
|
||||
|
||||
def test_onboarding_restart_uses_existing_no_resume_flags_and_exit_signal(startup_controls, monkeypatch):
|
||||
@pytest.fixture(params=['absent', 'initializing_at_admission', 'initializing_after_admission', 'dead_with_stale_bridge'])
|
||||
def startup_without_consumer(startup_controls, monkeypatch, request):
|
||||
"""A real thread is not a consumer until it has a published bridge."""
|
||||
from supervisor import message_bus
|
||||
|
||||
obj = startup_controls
|
||||
release = threading.Event()
|
||||
thread = threading.Thread(target=release.wait)
|
||||
bridge = message_bus.LocalChatBridge()
|
||||
if request.param == 'initializing_at_admission':
|
||||
thread.start()
|
||||
monkeypatch.setattr(obj.server, '_supervisor_thread', thread)
|
||||
|
||||
def admit(command):
|
||||
action = obj.server._startup_owner_command(command)
|
||||
if request.param in {'initializing_after_admission', 'dead_with_stale_bridge'}:
|
||||
thread.start()
|
||||
monkeypatch.setattr(obj.server, '_supervisor_thread', thread)
|
||||
if request.param == 'dead_with_stale_bridge':
|
||||
release.set()
|
||||
thread.join(timeout=2)
|
||||
assert not thread.is_alive()
|
||||
monkeypatch.setattr(message_bus, '_BRIDGE', bridge)
|
||||
return action
|
||||
|
||||
obj.app.state.startup_owner_command = admit
|
||||
try:
|
||||
yield obj
|
||||
assert bridge.get_updates(0, timeout=0) == [], 'a stale bridge must not receive the command'
|
||||
finally:
|
||||
release.set()
|
||||
if thread.ident is not None:
|
||||
thread.join(timeout=2)
|
||||
assert not thread.is_alive()
|
||||
|
||||
|
||||
def test_onboarding_restart_uses_existing_no_resume_flags_and_exit_signal(startup_without_consumer, monkeypatch):
|
||||
obj = startup_without_consumer
|
||||
calls = []
|
||||
monkeypatch.setattr(obj.server, '_safe_restart_serialized',
|
||||
from ouroboros import server_restart
|
||||
monkeypatch.setattr(server_restart, '_safe_restart_serialized',
|
||||
lambda fn, **kw: calls.append(('checked', kw)) or (True, 'ok'))
|
||||
monkeypatch.setattr(obj.server, '_stop_owned_work', lambda ctx: calls.append(('stopped', ctx.RUNNING)))
|
||||
monkeypatch.setattr(server_restart, '_stop_owned_work', lambda ctx: calls.append(('stopped', ctx.RUNNING)))
|
||||
response = obj.client.post('/api/command', json={'cmd': '/restart'})
|
||||
assert response.status_code == 200 and response.json() == {'status': 'ok'}
|
||||
assert calls == [('checked', {'reason': 'owner_restart', 'unsynced_policy': 'rescue_and_reset'}), ('stopped', {})]
|
||||
|
|
@ -81,10 +119,11 @@ def test_onboarding_restart_uses_existing_no_resume_flags_and_exit_signal(startu
|
|||
assert git_ops.REPO_DIR == obj.repo
|
||||
|
||||
|
||||
def test_onboarding_panic_runs_real_panic_marker_and_exit_99(startup_controls, monkeypatch):
|
||||
obj = startup_controls
|
||||
def test_onboarding_panic_runs_real_panic_marker_and_exit_99(startup_without_consumer, monkeypatch):
|
||||
obj = startup_without_consumer
|
||||
exits, stops = [], []
|
||||
from ouroboros import server_control
|
||||
from ouroboros.gateway.host_service import host_service_port
|
||||
monkeypatch.setattr('ouroboros.tools.shell.kill_all_tracked_subprocesses', lambda: None)
|
||||
monkeypatch.setattr('ouroboros.workspace_executor.kill_all_foreground', lambda *a, **kw: None)
|
||||
monkeypatch.setattr('ouroboros.tools.services.kill_all_services', lambda *a, **kw: None)
|
||||
|
|
@ -103,7 +142,9 @@ def test_onboarding_panic_runs_real_panic_marker_and_exit_99(startup_controls, m
|
|||
assert response.status_code == 200 and response.json() == {'status': 'ok'}
|
||||
assert exits == [99]
|
||||
assert (obj.data / 'state/panic_stop.flag').read_text(encoding="utf-8") == 'panic'
|
||||
assert ('daemon',) in stops and ('port', 19876) in stops
|
||||
assert stops == [('daemon',), ('workers', {'force': True, 'archive_service_logs': False,
|
||||
'reconcile_delegate_custody': False}),
|
||||
('port', 19876), ('port', host_service_port())]
|
||||
assert not (obj.data / 'settings.json').exists()
|
||||
from supervisor import state
|
||||
assert state.DRIVE_ROOT == obj.data
|
||||
|
|
@ -130,21 +171,46 @@ def test_configured_commands_keep_the_existing_bus_path(startup_controls, monkey
|
|||
assert not startup_controls.server._restart_requested.is_set()
|
||||
|
||||
|
||||
def test_onboarding_finishing_before_background_dispatch_reuses_live_consumer(startup_controls, monkeypatch):
|
||||
@pytest.mark.parametrize('command', ['/panic', '/restart'])
|
||||
def test_onboarding_finishing_before_background_dispatch_reuses_live_consumer(startup_controls, monkeypatch, command):
|
||||
from supervisor import message_bus
|
||||
|
||||
obj = startup_controls
|
||||
action = obj.server._startup_owner_command('/panic')
|
||||
calls = []
|
||||
monkeypatch.setattr(obj.server, '_supervisor_thread', SimpleNamespace(is_alive=lambda: True))
|
||||
monkeypatch.setattr('supervisor.message_bus._BRIDGE', SimpleNamespace(ui_send=lambda *a, **kw: calls.append((a, kw))))
|
||||
action()
|
||||
assert calls == [(('/panic',), {'broadcast': False})]
|
||||
release = threading.Event()
|
||||
thread = threading.Thread(target=release.wait)
|
||||
bridge = message_bus.LocalChatBridge()
|
||||
monkeypatch.setattr(message_bus, 'log_chat', lambda *a, **kw: None)
|
||||
monkeypatch.setattr(obj.server, '_execute_panic_stop', lambda *a: pytest.fail('duplicate direct Panic'))
|
||||
monkeypatch.setattr(obj.server, '_perform_owner_restart', lambda *a: pytest.fail('duplicate direct Restart'))
|
||||
|
||||
def admit(cmd):
|
||||
action = obj.server._startup_owner_command(cmd)
|
||||
thread.start()
|
||||
monkeypatch.setattr(obj.server, '_supervisor_thread', thread)
|
||||
monkeypatch.setattr(message_bus, '_BRIDGE', bridge)
|
||||
return action
|
||||
|
||||
obj.app.state.startup_owner_command = admit
|
||||
try:
|
||||
response = obj.client.post('/api/command', json={'cmd': command})
|
||||
assert response.status_code == 200 and response.json() == {'status': 'ok'}
|
||||
updates = bridge.get_updates(0, timeout=0)
|
||||
assert len(updates) == 1 and updates[0]['message']['text'] == command
|
||||
assert bridge.get_updates(0, timeout=0) == []
|
||||
assert not obj.server._restart_requested.is_set()
|
||||
finally:
|
||||
release.set()
|
||||
if thread.ident is not None:
|
||||
thread.join(timeout=2)
|
||||
assert not thread.is_alive()
|
||||
assert not (obj.data / 'state/panic_stop.flag').exists()
|
||||
|
||||
|
||||
def test_onboarding_restart_refusal_keeps_core_and_source(startup_controls, monkeypatch, caplog):
|
||||
obj = startup_controls
|
||||
monkeypatch.setattr(obj.server, '_safe_restart_serialized', lambda *a, **kw: (False, 'update is still resolving'))
|
||||
monkeypatch.setattr(obj.server, '_stop_owned_work', lambda ctx: pytest.fail('stopped before gate'))
|
||||
from ouroboros import server_restart
|
||||
monkeypatch.setattr(server_restart, '_safe_restart_serialized', lambda *a, **kw: (False, 'update is still resolving'))
|
||||
monkeypatch.setattr(server_restart, '_stop_owned_work', lambda ctx: pytest.fail('stopped before gate'))
|
||||
response = obj.client.post('/api/command', json={'cmd': '/restart'})
|
||||
assert response.status_code == 200 # accepted dispatch; failure remains explicit in lifecycle log
|
||||
assert not obj.server._restart_requested.is_set()
|
||||
|
|
|
|||
|
|
@ -377,7 +377,9 @@ def test_owner_restart_copy_is_explicit_about_stopped_task(tmp_path, monkeypatch
|
|||
from types import SimpleNamespace
|
||||
import server
|
||||
|
||||
source = _read("server.py")
|
||||
from ouroboros import server_restart
|
||||
|
||||
source = _read("ouroboros/server_restart.py")
|
||||
assert "Stopping active task. New settings apply to the next message." in source
|
||||
assert "owner_restart_no_resume.flag" in source
|
||||
assert "owner_restart_no_resume" in source
|
||||
|
|
@ -386,7 +388,7 @@ def test_owner_restart_copy_is_explicit_about_stopped_task(tmp_path, monkeypatch
|
|||
assert "stable_skip_flag.unlink(missing_ok=True)" in source
|
||||
# Checkout gate first (a refusal leaves the server intact), then the durable
|
||||
# no-resume intent, then the owned-work stop, then the owner's stop notice.
|
||||
owner_restart = source.split('elif lowered.startswith("/restart"):', 1)[1].split(
|
||||
owner_restart = _read("server.py").split('elif lowered.startswith("/restart"):', 1)[1].split(
|
||||
'elif lowered == "/review"', 1
|
||||
)[0]
|
||||
assert "_perform_owner_restart(ctx, reply)" in owner_restart
|
||||
|
|
@ -412,10 +414,10 @@ def test_owner_restart_copy_is_explicit_about_stopped_task(tmp_path, monkeypatch
|
|||
assert text == "Stopping active task. New settings apply to the next message."
|
||||
calls.append("notice")
|
||||
|
||||
monkeypatch.setattr(server, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(server, "_safe_restart_serialized", checked)
|
||||
monkeypatch.setattr(server, "_stop_owned_work", stopped)
|
||||
monkeypatch.setattr(server, "_request_restart_exit", lambda owner: calls.append(("exit", owner)))
|
||||
monkeypatch.setattr(server_restart, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(server_restart, "_safe_restart_serialized", checked)
|
||||
monkeypatch.setattr(server_restart, "_stop_owned_work", stopped)
|
||||
monkeypatch.setattr(server_restart, "_request_restart_exit", lambda owner: calls.append(("exit", owner)))
|
||||
assert server._perform_owner_restart(ctx, notice) == (True, "")
|
||||
assert calls == ["checked", "stopped", "notice", ("exit", True)]
|
||||
stop = _read("ouroboros/server_restart.py").split("def _stop_owned_work", 1)[1]
|
||||
|
|
@ -511,7 +513,7 @@ def test_owner_restart_proceeds_when_worker_shutdown_fails(tmp_path, monkeypatch
|
|||
monkeypatch.setattr(server_restart, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(config, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(message_bus, "log_chat", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(server, "_request_restart_exit", lambda owner=False: exits.append(owner))
|
||||
monkeypatch.setattr(server_restart, "_request_restart_exit", lambda owner=False: exits.append(owner))
|
||||
|
||||
server._process_bridge_updates(Bridge(), 0, Ctx())
|
||||
|
||||
|
|
@ -610,8 +612,10 @@ def test_only_an_owner_restart_asks_for_the_runtime_mode_to_be_re_read(tmp_path,
|
|||
|
||||
monkeypatch.setattr(server, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(message_bus, "log_chat", lambda *args, **kwargs: None)
|
||||
from ouroboros import server_restart
|
||||
monkeypatch.setattr(server_restart, "DATA_DIR", tmp_path)
|
||||
# The owned-work stop has its own suite; this pin is about the owner flag alone.
|
||||
monkeypatch.setattr(server, "_stop_owned_work", lambda ctx: None)
|
||||
monkeypatch.setattr(server_restart, "_stop_owned_work", lambda ctx: None)
|
||||
|
||||
server._owner_restart_requested.clear()
|
||||
server._restart_requested.clear()
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ _MOVED_OWNERS = {
|
|||
"_startup_worktree_prune": server_maintenance,
|
||||
"_live_running_task_ids": server_restart,
|
||||
"_managed_update_pending_kwargs": server_restart,
|
||||
"_perform_owner_restart": server_restart,
|
||||
"_safe_restart_serialized": server_restart,
|
||||
"_shutdown_supervisor_event_bus": server_restart,
|
||||
"_shutdown_task_cleanup_args": server_restart,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue