{
  "inspected": "2026-09-28",
  "status": "Historical saved evaluations; not a fresh inference run or a release certification.",
  "baseline": "Sarvam-M (corrected KV-cache baseline for GSM8K)",
  "candidate": "Stage R v8 (Sarvam-M fine-tune)",
  "suites": [
    {
      "suite": "gsm8k-indic",
      "baseline": {
        "label": "base_sarvam_m_cached",
        "suite": "gsm8k-indic",
        "n": 1300,
        "correct": 963,
        "accuracy": 0.7407692307692307,
        "serving": "unsloth-4bit",
        "think": false,
        "manifest": "sha256:4925f3fc8ee4a3d994c7291fad8a3b370522347efe69bf2ac86acfa0bf7f7cbb",
        "slices": {
          "gu_roman": {
            "correct": 70,
            "n": 100,
            "acc": 0.7
          },
          "mr_roman": {
            "correct": 62,
            "n": 100,
            "acc": 0.62
          },
          "te_roman": {
            "correct": 71,
            "n": 100,
            "acc": 0.71
          },
          "bn": {
            "correct": 86,
            "n": 100,
            "acc": 0.86
          },
          "gu": {
            "correct": 79,
            "n": 100,
            "acc": 0.79
          },
          "ta_roman": {
            "correct": 56,
            "n": 100,
            "acc": 0.56
          },
          "en": {
            "correct": 80,
            "n": 100,
            "acc": 0.8
          },
          "bn_roman": {
            "correct": 71,
            "n": 100,
            "acc": 0.71
          },
          "mr": {
            "correct": 75,
            "n": 100,
            "acc": 0.75
          },
          "ta": {
            "correct": 76,
            "n": 100,
            "acc": 0.76
          },
          "hi_roman": {
            "correct": 75,
            "n": 100,
            "acc": 0.75
          },
          "te": {
            "correct": 78,
            "n": 100,
            "acc": 0.78
          },
          "hi": {
            "correct": 84,
            "n": 100,
            "acc": 0.84
          }
        }
      },
      "candidate": {
        "label": "stage_r_v8",
        "suite": "gsm8k-indic",
        "n": 1300,
        "correct": 1092,
        "accuracy": 0.84,
        "serving": "unsloth-4bit",
        "think": false,
        "manifest": "sha256:4925f3fc8ee4a3d994c7291fad8a3b370522347efe69bf2ac86acfa0bf7f7cbb",
        "slices": {
          "gu_roman": {
            "correct": 76,
            "n": 100,
            "acc": 0.76
          },
          "mr_roman": {
            "correct": 84,
            "n": 100,
            "acc": 0.84
          },
          "te_roman": {
            "correct": 81,
            "n": 100,
            "acc": 0.81
          },
          "bn": {
            "correct": 95,
            "n": 100,
            "acc": 0.95
          },
          "gu": {
            "correct": 88,
            "n": 100,
            "acc": 0.88
          },
          "ta_roman": {
            "correct": 66,
            "n": 100,
            "acc": 0.66
          },
          "en": {
            "correct": 84,
            "n": 100,
            "acc": 0.84
          },
          "bn_roman": {
            "correct": 79,
            "n": 100,
            "acc": 0.79
          },
          "mr": {
            "correct": 88,
            "n": 100,
            "acc": 0.88
          },
          "ta": {
            "correct": 90,
            "n": 100,
            "acc": 0.9
          },
          "hi_roman": {
            "correct": 87,
            "n": 100,
            "acc": 0.87
          },
          "te": {
            "correct": 82,
            "n": 100,
            "acc": 0.82
          },
          "hi": {
            "correct": 92,
            "n": 100,
            "acc": 0.92
          }
        }
      }
    },
    {
      "suite": "mmlu-indic",
      "baseline": {
        "label": "base_sarvam_m",
        "suite": "mmlu-indic",
        "n": 2600,
        "correct": 1355,
        "accuracy": 0.5211538461538462,
        "serving": "unsloth-4bit",
        "think": false,
        "manifest": "sha256:0e0b7e55a97881a5fb8be8052c378064267fb408e102f0369515bc74f2cdd24a",
        "slices": {
          "bn": {
            "correct": 113,
            "n": 200,
            "acc": 0.565
          },
          "bn_roman": {
            "correct": 89,
            "n": 200,
            "acc": 0.445
          },
          "en": {
            "correct": 148,
            "n": 200,
            "acc": 0.74
          },
          "gu": {
            "correct": 91,
            "n": 200,
            "acc": 0.455
          },
          "gu_roman": {
            "correct": 80,
            "n": 200,
            "acc": 0.4
          },
          "hi": {
            "correct": 117,
            "n": 200,
            "acc": 0.585
          },
          "hi_roman": {
            "correct": 110,
            "n": 200,
            "acc": 0.55
          },
          "mr": {
            "correct": 106,
            "n": 200,
            "acc": 0.53
          },
          "mr_roman": {
            "correct": 106,
            "n": 200,
            "acc": 0.53
          },
          "ta": {
            "correct": 107,
            "n": 200,
            "acc": 0.535
          },
          "ta_roman": {
            "correct": 76,
            "n": 200,
            "acc": 0.38
          },
          "te": {
            "correct": 104,
            "n": 200,
            "acc": 0.52
          },
          "te_roman": {
            "correct": 108,
            "n": 200,
            "acc": 0.54
          }
        }
      },
      "candidate": {
        "label": "stage_r_v8",
        "suite": "mmlu-indic",
        "n": 2600,
        "correct": 1403,
        "accuracy": 0.5396153846153846,
        "serving": "unsloth-4bit",
        "think": false,
        "manifest": "sha256:0e0b7e55a97881a5fb8be8052c378064267fb408e102f0369515bc74f2cdd24a",
        "slices": {
          "bn": {
            "correct": 115,
            "n": 200,
            "acc": 0.575
          },
          "ta_roman": {
            "correct": 84,
            "n": 200,
            "acc": 0.42
          },
          "te": {
            "correct": 95,
            "n": 200,
            "acc": 0.475
          },
          "en": {
            "correct": 151,
            "n": 200,
            "acc": 0.755
          },
          "gu": {
            "correct": 104,
            "n": 200,
            "acc": 0.52
          },
          "mr": {
            "correct": 107,
            "n": 200,
            "acc": 0.535
          },
          "te_roman": {
            "correct": 112,
            "n": 200,
            "acc": 0.56
          },
          "gu_roman": {
            "correct": 87,
            "n": 200,
            "acc": 0.435
          },
          "hi_roman": {
            "correct": 114,
            "n": 200,
            "acc": 0.57
          },
          "mr_roman": {
            "correct": 106,
            "n": 200,
            "acc": 0.53
          },
          "ta": {
            "correct": 108,
            "n": 200,
            "acc": 0.54
          },
          "bn_roman": {
            "correct": 99,
            "n": 200,
            "acc": 0.495
          },
          "hi": {
            "correct": 121,
            "n": 200,
            "acc": 0.605
          }
        }
      }
    }
  ],
  "sources": [
    {
      "path": "aditya/eval/results/base_sarvam_m_cached_gsm8k-indic.json",
      "sha256": "be9f3b87cfb491b23682beb4b8d1f71ea019184bd8ebea04c75a62e980920cab"
    },
    {
      "path": "aditya/eval/results/stage_r_v8_gsm8k-indic.json",
      "sha256": "9953156a2891fcc5035c4364ef0d9573f2530de93ff1dda0255fb9012180023b"
    },
    {
      "path": "aditya/eval/results/base_sarvam_m_mmlu-indic.json",
      "sha256": "00764c296af165dfd3e0f84d6a0407efbc6ba4a7f15bb5584278a25757cd437d"
    },
    {
      "path": "aditya/eval/results/stage_r_v8_mmlu-indic.json",
      "sha256": "3cc5cef8c841fff3a2e64ea2c33a7ab688467c66ff225946f438d972e995d7c8"
    }
  ],
  "limitations": [
    "Repeated evaluation across development arms can affect selection; this is not a new blind holdout.",
    "Translated and Romanized items are not all statistically independent.",
    "No confidence intervals or broad safety claim are inferred from these two accuracy suites.",
    "Historic output-token limits are not fully recorded in every result file; do not infer them from current runner defaults.",
    "MMLU baseline serving and thinking fields were backfilled in the stored artifact.",
    "Accuracy is task-specific; Telugu native-script MMLU-Indic regressed from 52.0% to 47.5%."
  ]
}