mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-10-31 19:01:52 +00:00
38ba7a4007
The tested list held only the ladder's 8-case rows. Two suites now add rows after them: openrouter-screen v1, two cases after a smoke check, at agent when both pass and studio_only otherwise with its counts; and assumed, an expensive flagship listed at agent without runs, 0 of 0, which capability_of reports with reason assumed instead of counts. scripts/catalog_rows.py writes them from the sweep's sweep-results.json, a model the list already names keeping its row by its key or by a server's spelling. write_capability_list.py keeps them when it rewrites the ladder's rows. The real results come later; a fixture stands in.
35 lines
1.4 KiB
Python
35 lines
1.4 KiB
Python
"""Write the capability list the app ships from ladder results.
|
|
|
|
Run by hand after a ladder run is written up:
|
|
|
|
python scripts/write_capability_list.py [results.json]
|
|
|
|
With no argument it rewrites the list from the committed suite results in
|
|
`capability_list/ladder/`; rows from another suite, the screening sweep's
|
|
(`catalog_rows.py`), stay unless a ladder row names the model. The input's
|
|
shape is in `capability_list/ladder_input.py`; the bar each column is held to
|
|
is in `capability_list/verdict.py`. A person reviews both diffs and commits
|
|
them; nothing fetches the list at runtime.
|
|
"""
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
from capability_list.ladder_input import LadderResults
|
|
from capability_list.rows import COMMITTED_INPUT, measured_rows, with_ladder_rows
|
|
from capability_list.shipped import read_shipped, write_shipped
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("results", nargs="?", type=Path, default=COMMITTED_INPUT)
|
|
args = parser.parse_args()
|
|
results = LadderResults.model_validate_json(args.results.read_text("utf-8"))
|
|
written = with_ladder_rows(read_shipped(), measured_rows(results))
|
|
write_shipped(written)
|
|
for row in written.models:
|
|
print(f"{row.key}: {row.level} ({row.passes.passed}/{row.passes.counted})")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|