[
 {
  "file": "nemotron-3-ultra-550b-a55b_T1_A_r1.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T1",
  "arm": "A",
  "run": 1,
  "loc": 1,
  "completion_tokens": 328,
  "checks": [
   0,
   9
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "CHECKS 0 9\nBLOCKERR SyntaxError invalid syntax (block, line 1)\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T1_P_r3.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T1",
  "arm": "P",
  "run": 3,
  "loc": 4,
  "completion_tokens": 320,
  "checks": [
   9,
   9
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T1_P_r4.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T1",
  "arm": "P",
  "run": 4,
  "loc": 4,
  "completion_tokens": 343,
  "checks": [
   9,
   9
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_A_r1.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "A",
  "run": 1,
  "loc": 11,
  "completion_tokens": 331,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_A_r2.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "A",
  "run": 2,
  "loc": 10,
  "completion_tokens": 513,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_A_r3.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "A",
  "run": 3,
  "loc": 0,
  "completion_tokens": 67,
  "checks": [
   2,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "okerr name 'save_upload' is not defined\nCHECKS 2 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_A_r4.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "A",
  "run": 4,
  "loc": 0,
  "completion_tokens": 109,
  "checks": [
   2,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "okerr name 'save_upload' is not defined\nCHECKS 2 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_B_r3.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "B",
  "run": 3,
  "loc": 19,
  "completion_tokens": 719,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_B_r4.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "B",
  "run": 4,
  "loc": 19,
  "completion_tokens": 418,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_B_r5.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "B",
  "run": 5,
  "loc": 59,
  "completion_tokens": 1200,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r3.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 3,
  "loc": 7,
  "completion_tokens": 147,
  "checks": [
   1,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "CHECKS 1 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r4.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 4,
  "loc": 7,
  "completion_tokens": 234,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r5.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 5,
  "loc": 8,
  "completion_tokens": 238,
  "checks": [
   1,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "CHECKS 1 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r6.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 6,
  "loc": 7,
  "completion_tokens": 207,
  "checks": [
   1,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "CHECKS 1 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r7.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 7,
  "loc": 7,
  "completion_tokens": 173,
  "checks": [
   3,
   3
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T4_P_r8.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T4",
  "arm": "P",
  "run": 8,
  "loc": 5,
  "completion_tokens": 131,
  "checks": [
   1,
   3
  ],
  "pass": false,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": "CHECKS 1 3\n"
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T5_A_r1.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T5",
  "arm": "A",
  "run": 1,
  "loc": 9,
  "completion_tokens": 368,
  "checks": [
   5,
   5
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T5_P_r3.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T5",
  "arm": "P",
  "run": 3,
  "loc": 2,
  "completion_tokens": 145,
  "checks": [
   5,
   5
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 },
 {
  "file": "nemotron-3-ultra-550b-a55b_T5_P_r4.json",
  "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
  "task": "T5",
  "arm": "P",
  "run": 4,
  "loc": 8,
  "completion_tokens": 239,
  "checks": [
   5,
   5
  ],
  "pass": true,
  "has_test_strict": false,
  "has_test_broad": false,
  "finish": "stop",
  "log_tail": ""
 }
]