Studio: head a decision score with its most likely level (#13111)

* Studio: head a decision score with its most likely level

The Try a decision dialog named the level nearest the expected score, so a
split answer (83% on 0, 17% on 3, expectation 0.50) was headed with level 1,
which the model gave 0.2%. Use the most likely level, falling back to the
nearest when no probabilities came back.

* test(studio): pin a decision score's top level away from the first level and on ties

The split-score case expected level 0, which an implementation returning the first level whenever probabilities exist also satisfies, and nothing covered a tie. Add a mirrored split that must head with level 3 and a flat distribution that must keep the nearest level. Also shorten the comment in scoreLevels and wrap the reduce the way the formatter does.

---------

Co-authored-by: Lyxot <longyixing331@gmail.com>

studio/frontend/src/features/settings/components/decision-try-dialog.tsx

@@ -601,11 +601,11 @@ function AnswerPreview({
       </div>
     );
   }
-  const { keys, nearest, max } = scoreLevels(answer);
+  const { keys, top, max } = scoreLevels(answer);
   return (
     <div className="flex flex-col gap-6">
       <AnswerHeading
-        answer={criterionText(answer.legend[nearest]) ?? nearest}
+        answer={criterionText(answer.legend[top]) ?? top}
         detail={t("decisions.scoreDetail", {
           score: answer.score.toFixed(2),
           max,
@@ -618,7 +618,7 @@ function AnswerPreview({
             index={key}
             label={criterionText(answer.legend[key]) ?? key}
             value={answer.probabilities[key] ?? 0}
-            top={key === nearest}
+            top={key === top}
             percent={percent}
           />
         ))}

studio/frontend/src/features/settings/lib/decision-request.ts

@@ -375,12 +375,18 @@ export function choiceCriterion(
 
 export function scoreLevels(
   answer: Extract<DecisionAnswer, { type: "score" }>,
-): { keys: string[]; nearest: string; max: number } {
+): { keys: string[]; top: string; max: number } {
   const keys = Object.keys(answer.legend).sort((a, b) => Number(a) - Number(b));
   const distance = (key: string) => Math.abs(Number(key) - answer.score);
   const nearest = keys.reduce(
     (best, key) => (distance(key) < distance(best) ? key : best),
     keys[0] ?? "0",
   );
-  return { keys, nearest, max: Number(keys.at(-1) ?? 0) };
+  // The score is an expectation, so a split answer's nearest level can be one the model barely picked.
+  const p = (key: string) => answer.probabilities[key] ?? -1;
+  const top = keys.reduce(
+    (best, key) => (p(key) > p(best) ? key : best),
+    nearest,
+  );
+  return { keys, top, max: Number(keys.at(-1) ?? 0) };
 }

studio/frontend/tests/decisions-request.test.ts

@@ -165,7 +165,7 @@ test("every type starts from its first preset and every preset is a valid reques
   }
 });
 
-test("a score lands on the nearest level of its legend", () => {
+test("a score with no probabilities lands on the nearest level of its legend", () => {
   const answer = {
     type: "score" as const,
     score: 2.6,
@@ -175,10 +175,32 @@ test("a score lands on the nearest level of its legend", () => {
   };
   assert.deepEqual(scoreLevels(answer), {
     keys: ["0", "1", "2", "10"],
-    nearest: "2",
+    top: "2",
     max: 10,
   });
-  assert.equal(scoreLevels({ ...answer, score: -1 }).nearest, "0");
+  assert.equal(scoreLevels({ ...answer, score: -1 }).top, "0");
+});
+
+test("a split score heads with its most likely level, not the one nearest the expectation", () => {
+  // Measured in Unsloth Desktop 0.1.905: the 429 grading example came back like this.
+  const answer = {
+    type: "score" as const,
+    score: 0.5032,
+    confidence: 0.6559,
+    legend: { "0": "wrong", "1": "touches", "2": "partial", "3": "correct" },
+    probabilities: { "0": 0.8303, "1": 0.0021, "2": 0.0018, "3": 0.1658 },
+  };
+  assert.equal(scoreLevels(answer).top, "0");
+  const mirrored = { "0": 0.1658, "1": 0.0018, "2": 0.0021, "3": 0.8303 };
+  assert.equal(
+    scoreLevels({ ...answer, score: 2.4968, probabilities: mirrored }).top,
+    "3",
+  );
+  const flat = { "0": 0.25, "1": 0.25, "2": 0.25, "3": 0.25 };
+  assert.equal(
+    scoreLevels({ ...answer, score: 1.5, probabilities: flat }).top,
+    "1",
+  );
 });
 
 test("Try it in Settings > API opens the decision playground", () => {