{
 "model": "qwen2.5-1.5b-instruct-q4_k_m.gguf sha256 6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e",
 "slices": {
  "synthetic_logs_slice": {
   "tokens_before": 11477,
   "tokens_after": 5161,
   "transforms": [
    "router:protected:user_message",
    "router:mixed:0.32"
   ],
   "ccr_markers": 0,
   "questions": [
    {
     "input": "original",
     "q": "What is the request_id of the FATAL entry?",
     "truth": "req-424242",
     "model": "424242",
     "correct": false,
     "prompt_tokens": 13802,
     "seconds": 76.6
    },
    {
     "input": "original",
     "q": "Which service logged the FATAL entry?",
     "truth": "billing",
     "model": "billing",
     "correct": true,
     "prompt_tokens": 13799,
     "seconds": 1.1
    },
    {
     "input": "original",
     "q": "What is the latency_ms of request req-538890?",
     "truth": "597",
     "model": "597",
     "correct": true,
     "prompt_tokens": 13807,
     "seconds": 1.6
    },
    {
     "input": "original",
     "q": "What is the level of request req-861033?",
     "truth": "INFO",
     "model": "INFO",
     "correct": true,
     "prompt_tokens": 13806,
     "seconds": 1.1
    },
    {
     "input": "original",
     "q": "Which service handled request req-157953?",
     "truth": "search",
     "model": "search",
     "correct": true,
     "prompt_tokens": 13804,
     "seconds": 1.2
    },
    {
     "input": "compressed",
     "q": "What is the request_id of the FATAL entry?",
     "truth": "req-424242",
     "model": "424242",
     "correct": false,
     "prompt_tokens": 7455,
     "seconds": 30.8
    },
    {
     "input": "compressed",
     "q": "Which service logged the FATAL entry?",
     "truth": "billing",
     "model": "ledger",
     "correct": false,
     "prompt_tokens": 7452,
     "seconds": 0.8
    },
    {
     "input": "compressed",
     "q": "What is the latency_ms of request req-538890?",
     "truth": "597",
     "model": "593",
     "correct": false,
     "prompt_tokens": 7460,
     "seconds": 1.3
    },
    {
     "input": "compressed",
     "q": "What is the level of request req-861033?",
     "truth": "INFO",
     "model": "INFO",
     "correct": true,
     "prompt_tokens": 7459,
     "seconds": 0.9
    },
    {
     "input": "compressed",
     "q": "Which service handled request req-157953?",
     "truth": "search",
     "model": "search",
     "correct": true,
     "prompt_tokens": 7457,
     "seconds": 0.7
    }
   ],
   "original_correct": 4,
   "compressed_correct": 2,
   "same_answer_both": 3
  },
  "wiki_search_slice": {
   "tokens_before": 12788,
   "tokens_after": 7953,
   "transforms": [
    "router:protected:user_message",
    "router:mixed:0.53"
   ],
   "ccr_markers": 15,
   "questions": [
    {
     "input": "original",
     "q": "What is the pageid of the article titled 'Python (codename)'?",
     "truth": "53672527",
     "model": "53672527",
     "correct": true,
     "prompt_tokens": 14688,
     "seconds": 84.0
    },
    {
     "input": "original",
     "q": "What is the wordcount of the article titled 'Python (mythology)'?",
     "truth": "1218",
     "model": "1218",
     "correct": true,
     "prompt_tokens": 14689,
     "seconds": 1.9
    },
    {
     "input": "original",
     "q": "What is the title of the article with pageid 40170617?",
     "truth": "2013 New Brunswick python attack",
     "model": "2013 New Brunswick python attack",
     "correct": true,
     "prompt_tokens": 14692,
     "seconds": 2.8
    },
    {
     "input": "original",
     "q": "What is the size of the article titled 'IDLE'?",
     "truth": "5607",
     "model": "5607",
     "correct": true,
     "prompt_tokens": 14685,
     "seconds": 1.8
    },
    {
     "input": "original",
     "q": "What is the title of the very first result?",
     "truth": "Python (programming language)",
     "model": "Python (programming language)",
     "correct": true,
     "prompt_tokens": 14682,
     "seconds": 1.6
    },
    {
     "input": "compressed",
     "q": "What is the pageid of the article titled 'Python (codename)'?",
     "truth": "53672527",
     "model": "23862",
     "correct": false,
     "prompt_tokens": 9822,
     "seconds": 44.7
    },
    {
     "input": "compressed",
     "q": "What is the wordcount of the article titled 'Python (mythology)'?",
     "truth": "1218",
     "model": "1218",
     "correct": true,
     "prompt_tokens": 9823,
     "seconds": 1.4
    },
    {
     "input": "compressed",
     "q": "What is the title of the article with pageid 40170617?",
     "truth": "2013 New Brunswick python attack",
     "model": "Michael Palin",
     "correct": false,
     "prompt_tokens": 9826,
     "seconds": 1.2
    },
    {
     "input": "compressed",
     "q": "What is the size of the article titled 'IDLE'?",
     "truth": "5607",
     "model": "17906",
     "correct": false,
     "prompt_tokens": 9819,
     "seconds": 1.5
    },
    {
     "input": "compressed",
     "q": "What is the title of the very first result?",
     "truth": "Python (programming language)",
     "model": "Python (programming language)",
     "correct": true,
     "prompt_tokens": 9816,
     "seconds": 1.1
    }
   ],
   "original_correct": 5,
   "compressed_correct": 2,
   "same_answer_both": 2
  }
 },
 "summary": {
  "questions_per_input": 10,
  "original_correct": 9,
  "compressed_correct": 4,
  "identical_answers": 5
 }
}