{
 "model": "qwen2.5-3b-instruct-q4_k_m.gguf sha256 626b4a6678b86442240e33df819e00132d3ba7dddfe1cdc4fbb18e0a9615c62d",
 "slices": {
  "synthetic_logs_slice": {
   "tokens_before": 11477,
   "tokens_after": 5161,
   "transforms": [
    "router:protected:user_message",
    "router:mixed:0.32"
   ],
   "ccr_markers": 0,
   "questions": [
    {
     "input": "original",
     "q": "What is the request_id of the FATAL entry?",
     "truth": "req-424242",
     "model": "req-424242",
     "correct": true,
     "prompt_tokens": 13802,
     "seconds": 142.2
    },
    {
     "input": "original",
     "q": "Which service logged the FATAL entry?",
     "truth": "billing",
     "model": "billing",
     "correct": true,
     "prompt_tokens": 13799,
     "seconds": 1.6
    },
    {
     "input": "original",
     "q": "What is the latency_ms of request req-538890?",
     "truth": "597",
     "model": "597",
     "correct": true,
     "prompt_tokens": 13807,
     "seconds": 2.5
    },
    {
     "input": "original",
     "q": "What is the level of request req-861033?",
     "truth": "INFO",
     "model": "INFO",
     "correct": true,
     "prompt_tokens": 13806,
     "seconds": 1.9
    },
    {
     "input": "original",
     "q": "Which service handled request req-157953?",
     "truth": "search",
     "model": "search",
     "correct": true,
     "prompt_tokens": 13804,
     "seconds": 1.9
    },
    {
     "input": "compressed",
     "q": "What is the request_id of the FATAL entry?",
     "truth": "req-424242",
     "model": "req-424242",
     "correct": true,
     "prompt_tokens": 7455,
     "seconds": 56.4
    },
    {
     "input": "compressed",
     "q": "Which service logged the FATAL entry?",
     "truth": "billing",
     "model": "ledger",
     "correct": false,
     "prompt_tokens": 7452,
     "seconds": 1.2
    },
    {
     "input": "compressed",
     "q": "What is the latency_ms of request req-538890?",
     "truth": "597",
     "model": "597",
     "correct": true,
     "prompt_tokens": 7460,
     "seconds": 1.7
    },
    {
     "input": "compressed",
     "q": "What is the level of request req-861033?",
     "truth": "INFO",
     "model": "INFO",
     "correct": true,
     "prompt_tokens": 7459,
     "seconds": 1.7
    },
    {
     "input": "compressed",
     "q": "Which service handled request req-157953?",
     "truth": "search",
     "model": "search",
     "correct": true,
     "prompt_tokens": 7457,
     "seconds": 1.7
    }
   ],
   "original_correct": 5,
   "compressed_correct": 4,
   "same_answer_both": 4
  },
  "wiki_search_slice": {
   "tokens_before": 12788,
   "tokens_after": 7953,
   "transforms": [
    "router:protected:user_message",
    "router:mixed:0.53"
   ],
   "ccr_markers": 15,
   "questions": [
    {
     "input": "original",
     "q": "What is the pageid of the article titled 'Python (codename)'?",
     "truth": "53672527",
     "model": "53672527",
     "correct": true,
     "prompt_tokens": 14688,
     "seconds": 146.5
    },
    {
     "input": "original",
     "q": "What is the wordcount of the article titled 'Python (mythology)'?",
     "truth": "1218",
     "model": "1218",
     "correct": true,
     "prompt_tokens": 14689,
     "seconds": 3.0
    },
    {
     "input": "original",
     "q": "What is the title of the article with pageid 40170617?",
     "truth": "2013 New Brunswick python attack",
     "model": "2013 New Brunswick python attack",
     "correct": true,
     "prompt_tokens": 14692,
     "seconds": 3.8
    },
    {
     "input": "original",
     "q": "What is the size of the article titled 'IDLE'?",
     "truth": "5607",
     "model": "5607",
     "correct": true,
     "prompt_tokens": 14685,
     "seconds": 2.6
    },
    {
     "input": "original",
     "q": "What is the title of the very first result?",
     "truth": "Python (programming language)",
     "model": "Python (programming language)",
     "correct": true,
     "prompt_tokens": 14682,
     "seconds": 2.7
    },
    {
     "input": "compressed",
     "q": "What is the pageid of the article titled 'Python (codename)'?",
     "truth": "53672527",
     "model": "46448252",
     "correct": false,
     "prompt_tokens": 9822,
     "seconds": 84.5
    },
    {
     "input": "compressed",
     "q": "What is the wordcount of the article titled 'Python (mythology)'?",
     "truth": "1218",
     "model": "1218",
     "correct": true,
     "prompt_tokens": 9823,
     "seconds": 2.5
    },
    {
     "input": "compressed",
     "q": "What is the title of the article with pageid 40170617?",
     "truth": "2013 New Brunswick python attack",
     "model": "The Borneo python attack",
     "correct": false,
     "prompt_tokens": 9826,
     "seconds": 2.9
    },
    {
     "input": "compressed",
     "q": "What is the size of the article titled 'IDLE'?",
     "truth": "5607",
     "model": "99",
     "correct": false,
     "prompt_tokens": 9819,
     "seconds": 1.6
    },
    {
     "input": "compressed",
     "q": "What is the title of the very first result?",
     "truth": "Python (programming language)",
     "model": "Monty Python",
     "correct": false,
     "prompt_tokens": 9816,
     "seconds": 2.3
    }
   ],
   "original_correct": 5,
   "compressed_correct": 1,
   "same_answer_both": 1
  }
 },
 "summary": {
  "questions_per_input": 10,
  "original_correct": 10,
  "compressed_correct": 5,
  "identical_answers": 5
 }
}