mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-19 10:04:16 +00:00
Keep CyberGym refusal manifests single-provider
Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
This commit is contained in:
parent
b6bc0ab66c
commit
8e6e528c57
3 changed files with 57 additions and 2 deletions
|
|
@ -619,6 +619,9 @@ def benchmark_run_manifest(
|
|||
meta_settings_path = meta.get("settings_path")
|
||||
if meta_settings_path is not None:
|
||||
meta_settings_path = pathlib.Path(meta_settings_path)
|
||||
include_claude_sdk_defaults = meta.get("include_claude_sdk_defaults", True)
|
||||
if not isinstance(include_claude_sdk_defaults, bool):
|
||||
raise ValueError("include_claude_sdk_defaults metadata must be a boolean")
|
||||
source = repo_provenance(repo_dir)
|
||||
gate = _seed_gate(source, require_clean=require_clean, expect=str(expect or ""))
|
||||
manifest: dict[str, Any] = {
|
||||
|
|
@ -636,7 +639,10 @@ def benchmark_run_manifest(
|
|||
"isolated_data_root": str(meta.get("isolated_data_root") or ""),
|
||||
"output_paths": meta.get("output_paths") or {},
|
||||
"model_slots": model_slot_snapshot(meta_settings_path),
|
||||
"provider_credentials": provider_credential_disclosure(meta_settings_path),
|
||||
"provider_credentials": provider_credential_disclosure(
|
||||
meta_settings_path,
|
||||
include_claude_sdk_defaults=include_claude_sdk_defaults,
|
||||
),
|
||||
"source": source,
|
||||
"seed_gate": gate,
|
||||
"extra": meta.get("extra") or {},
|
||||
|
|
|
|||
|
|
@ -1022,6 +1022,9 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
official_command=_generator_template(args),
|
||||
isolated_data_root=str(args.data_root or ""),
|
||||
settings_path=settings_path,
|
||||
# The applied CyberGym profile explicitly disables both Claude SDK
|
||||
# transports; keep even the pre-admission/refusal disclosure truthful.
|
||||
include_claude_sdk_defaults=False,
|
||||
output_paths={
|
||||
"run_root": str(out_root),
|
||||
"manifest": str(manifest_path),
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ import json
|
|||
import pathlib
|
||||
import re
|
||||
|
||||
from devtools.benchmarks.common.manifests import provider_credential_disclosure
|
||||
from devtools.benchmarks.common.manifests import (
|
||||
benchmark_run_manifest,
|
||||
provider_credential_disclosure,
|
||||
)
|
||||
from devtools.benchmarks.common.result_index import (
|
||||
RUNTIME_TRUNCATION_REASON_CODES,
|
||||
runtime_terminal_disclosure,
|
||||
|
|
@ -282,6 +285,49 @@ def test_manifest_discloses_runtime_injected_credentials_separately(tmp_path):
|
|||
assert "runtime-openai-value" not in blob
|
||||
|
||||
|
||||
def test_initial_manifest_uses_disabled_claude_projection(tmp_path, monkeypatch):
|
||||
"""A pre-application/refusal manifest must not reintroduce the Claude default projection."""
|
||||
settings_path = tmp_path / "settings.json"
|
||||
settings_path.write_text(
|
||||
json.dumps({
|
||||
"OUROBOROS_MODEL": "openrouter/model",
|
||||
"CLAUDE_CODE_MODEL": "",
|
||||
"CLAUDE_AGENT_SDK_MODEL": "",
|
||||
"OPENROUTER_API_KEY": "",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
import devtools.benchmarks.common.manifests as manifests
|
||||
|
||||
monkeypatch.setattr(
|
||||
manifests,
|
||||
"repo_provenance",
|
||||
lambda _path: {
|
||||
"repo_dir": str(tmp_path / "repo"),
|
||||
"git_available": True,
|
||||
"status_available": True,
|
||||
"dirty": False,
|
||||
"head": "a" * 40,
|
||||
"version": "",
|
||||
"describe": "a" * 40,
|
||||
},
|
||||
)
|
||||
manifest = benchmark_run_manifest(
|
||||
benchmark="cybergym",
|
||||
run_root=tmp_path / "run",
|
||||
repo_dir=tmp_path / "repo",
|
||||
requested_task_ids=["arvo:1"],
|
||||
metadata={
|
||||
"settings_path": settings_path,
|
||||
"include_claude_sdk_defaults": False,
|
||||
},
|
||||
)
|
||||
disclosure = manifest["provider_credentials"]
|
||||
assert disclosure["providers"] == {"openrouter": ["openrouter/model"]}
|
||||
assert disclosure["planned_keys"] == ["OPENROUTER_API_KEY"]
|
||||
assert "CLAUDE_CODE_MODEL" not in disclosure["declared_model_slots"]
|
||||
|
||||
|
||||
def test_isolated_credential_grants_reports_the_file_not_the_intent():
|
||||
"""`planned_keys` is the derivation; `granted` is the truth about the file. An explicit
|
||||
override the slots never asked for must be VISIBLE, not inferred away."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue