Keep CyberGym refusal manifests single-provider

Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
This commit is contained in:
Anton Razzhigaev 2026-08-28 02:47:39 +00:00
parent b6bc0ab66c
commit 8e6e528c57
3 changed files with 57 additions and 2 deletions

View file

@ -619,6 +619,9 @@ def benchmark_run_manifest(
meta_settings_path = meta.get("settings_path")
if meta_settings_path is not None:
meta_settings_path = pathlib.Path(meta_settings_path)
include_claude_sdk_defaults = meta.get("include_claude_sdk_defaults", True)
if not isinstance(include_claude_sdk_defaults, bool):
raise ValueError("include_claude_sdk_defaults metadata must be a boolean")
source = repo_provenance(repo_dir)
gate = _seed_gate(source, require_clean=require_clean, expect=str(expect or ""))
manifest: dict[str, Any] = {
@ -636,7 +639,10 @@ def benchmark_run_manifest(
"isolated_data_root": str(meta.get("isolated_data_root") or ""),
"output_paths": meta.get("output_paths") or {},
"model_slots": model_slot_snapshot(meta_settings_path),
"provider_credentials": provider_credential_disclosure(meta_settings_path),
"provider_credentials": provider_credential_disclosure(
meta_settings_path,
include_claude_sdk_defaults=include_claude_sdk_defaults,
),
"source": source,
"seed_gate": gate,
"extra": meta.get("extra") or {},

View file

@ -1022,6 +1022,9 @@ def main(argv: Sequence[str] | None = None) -> int:
official_command=_generator_template(args),
isolated_data_root=str(args.data_root or ""),
settings_path=settings_path,
# The applied CyberGym profile explicitly disables both Claude SDK
# transports; keep even the pre-admission/refusal disclosure truthful.
include_claude_sdk_defaults=False,
output_paths={
"run_root": str(out_root),
"manifest": str(manifest_path),

View file

@ -18,7 +18,10 @@ import json
import pathlib
import re
from devtools.benchmarks.common.manifests import provider_credential_disclosure
from devtools.benchmarks.common.manifests import (
benchmark_run_manifest,
provider_credential_disclosure,
)
from devtools.benchmarks.common.result_index import (
RUNTIME_TRUNCATION_REASON_CODES,
runtime_terminal_disclosure,
@ -282,6 +285,49 @@ def test_manifest_discloses_runtime_injected_credentials_separately(tmp_path):
assert "runtime-openai-value" not in blob
def test_initial_manifest_uses_disabled_claude_projection(tmp_path, monkeypatch):
"""A pre-application/refusal manifest must not reintroduce the Claude default projection."""
settings_path = tmp_path / "settings.json"
settings_path.write_text(
json.dumps({
"OUROBOROS_MODEL": "openrouter/model",
"CLAUDE_CODE_MODEL": "",
"CLAUDE_AGENT_SDK_MODEL": "",
"OPENROUTER_API_KEY": "",
}),
encoding="utf-8",
)
import devtools.benchmarks.common.manifests as manifests
monkeypatch.setattr(
manifests,
"repo_provenance",
lambda _path: {
"repo_dir": str(tmp_path / "repo"),
"git_available": True,
"status_available": True,
"dirty": False,
"head": "a" * 40,
"version": "",
"describe": "a" * 40,
},
)
manifest = benchmark_run_manifest(
benchmark="cybergym",
run_root=tmp_path / "run",
repo_dir=tmp_path / "repo",
requested_task_ids=["arvo:1"],
metadata={
"settings_path": settings_path,
"include_claude_sdk_defaults": False,
},
)
disclosure = manifest["provider_credentials"]
assert disclosure["providers"] == {"openrouter": ["openrouter/model"]}
assert disclosure["planned_keys"] == ["OPENROUTER_API_KEY"]
assert "CLAUDE_CODE_MODEL" not in disclosure["declared_model_slots"]
def test_isolated_credential_grants_reports_the_file_not_the_intent():
"""`planned_keys` is the derivation; `granted` is the truth about the file. An explicit
override the slots never asked for must be VISIBLE, not inferred away."""