{
  "study_id": "gemma-4-e2b-json-2026-09-15",
  "kind": "exploratory structured-output probe; not suite-v1 or a signed leaderboard run",
  "measured_on": "2026-09-15",
  "model": {
    "publisher": "google",
    "repository": "google/gemma-4-E2B-it-qat-q4_0-gguf",
    "revision": "675cff42a74c774d6cb76f76d8eacb49b48c9b93",
    "filename": "gemma-4-E2B_q4_0-it.gguf",
    "bytes": 3349516256,
    "sha256": "fa401b55b07ee70a54c6dae3903c783a6e65064312529ea57175cb5f8dec6634"
  },
  "runtime": {
    "name": "llama.cpp",
    "commit": "9725a313be0528214c4a02fed906ddaf7b3f712e",
    "hardware": "RTX 5090, WSL Linux",
    "settings": {
      "context": 4096,
      "parallel": 1,
      "gpu_layers_requested": 999,
      "batch_size": 128,
      "ubatch_size": 64,
      "flash_attention": "on",
      "jinja": true,
      "reasoning": "off",
      "cache_ram": 0
    },
    "co_resident_models": [
      "Qwen3.6-27B Q4_K_M coding service",
      "Qwen3.5-4B Q4_K_M planner service"
    ],
    "other_services": "resident and idle during checks; not stopped",
    "allocation_evidence": [
      "load_tensors: offloaded 36/36 layers to GPU",
      "load_tensors:   CPU_Mapped model buffer size =  2152.50 MiB",
      "load_tensors:        CUDA0 model buffer size =  1341.78 MiB",
      "llama_kv_cache:      CUDA0 KV buffer size =    24.00 MiB",
      "llama_kv_cache:      CUDA0 KV buffer size =     9.00 MiB",
      "sched_reserve:      CUDA0 compute buffer size =    64.38 MiB"
    ]
  },
  "protocol": {
    "cases": [
      {
        "name": "route",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write a Python function that reverses a list.",
        "expected": {
          "route": "code"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "extract",
        "prompt": "Return only JSON with model and memory_gb extracted from this text: I have an RTX 5090 with 32 GB VRAM.",
        "expected": {
          "model": "RTX 5090",
          "memory_gb": 32
        },
        "schema": {
          "type": "object",
          "properties": {
            "model": {
              "type": "string"
            },
            "memory_gb": {
              "type": "integer"
            }
          },
          "required": [
            "model",
            "memory_gb"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "missing",
        "prompt": "Return only JSON with price_usd extracted from this text: NVIDIA RTX 5090, 32 GB VRAM. Use null when the price is absent.",
        "expected": {
          "price_usd": null
        },
        "schema": {
          "type": "object",
          "properties": {
            "price_usd": {
              "type": [
                "number",
                "null"
              ]
            }
          },
          "required": [
            "price_usd"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-2",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write JavaScript that sorts an array of numbers.",
        "expected": {
          "route": "code"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-3",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Explain the difference between RAM and VRAM.",
        "expected": {
          "route": "lookup"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-4",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: What does GPU stand for?",
        "expected": {
          "route": "lookup"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-5",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: ",
        "expected": {
          "route": "clarify"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-6",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request:    ",
        "expected": {
          "route": "clarify"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      },
      {
        "name": "route-7",
        "prompt": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Implement binary search in Python.",
        "expected": {
          "route": "code"
        },
        "schema": {
          "type": "object",
          "properties": {
            "route": {
              "type": "string",
              "enum": [
                "code",
                "lookup",
                "clarify"
              ]
            }
          },
          "required": [
            "route"
          ],
          "additionalProperties": false
        }
      }
    ],
    "execution_order": [
      [
        "route-5",
        "schema"
      ],
      [
        "route-5",
        "prompt_only"
      ],
      [
        "missing",
        "prompt_only"
      ],
      [
        "route-3",
        "schema"
      ],
      [
        "missing",
        "schema"
      ],
      [
        "route-2",
        "schema"
      ],
      [
        "route-7",
        "prompt_only"
      ],
      [
        "route-2",
        "prompt_only"
      ],
      [
        "route-4",
        "prompt_only"
      ],
      [
        "route-6",
        "schema"
      ],
      [
        "route",
        "schema"
      ],
      [
        "route-4",
        "schema"
      ],
      [
        "extract",
        "prompt_only"
      ],
      [
        "route-6",
        "prompt_only"
      ],
      [
        "route-7",
        "schema"
      ],
      [
        "route-3",
        "prompt_only"
      ],
      [
        "route",
        "prompt_only"
      ],
      [
        "extract",
        "schema"
      ]
    ],
    "order_seed": 42,
    "repetitions_per_case_condition": 1,
    "prior_smoke_replies": 3,
    "prior_smoke_excluded_from_comparison": true,
    "strict_json": "json.loads on unmodified content; no fence stripping",
    "value_check": "Parsed object must equal the expected object exactly",
    "limitations": [
      "Nine hand-written, very short cases are not representative agent accuracy.",
      "One request per case per condition; not statistical confidence or a speed comparison.",
      "The main/helper concurrency hypothesis was deferred after a routing error.",
      "Temperature zero and disabled thinking describe this configuration, not all settings.",
      "All-layer offload is not all weights in VRAM: runtime retains a CPU-mapped model buffer."
    ]
  },
  "results": [
    {
      "case": "route-5",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: "
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "clarify"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"clarify\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 18,
          "prompt_tokens": 42,
          "total_tokens": 60,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473750
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route-5",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: "
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "clarify"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"clarification\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 18,
          "prompt_tokens": 42,
          "total_tokens": 60,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473750
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "missing",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only JSON with price_usd extracted from this text: NVIDIA RTX 5090, 32 GB VRAM. Use null when the price is absent."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "price_usd": null
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"price_usd\": null\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 44,
          "total_tokens": 61,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473750
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-3",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Explain the difference between RAM and VRAM."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "lookup"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"code\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 51,
          "total_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": true,
      "exact_value_pass": false
    },
    {
      "case": "missing",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only JSON with price_usd extracted from this text: NVIDIA RTX 5090, 32 GB VRAM. Use null when the price is absent."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "price_usd": {
                "type": [
                  "number",
                  "null"
                ]
              }
            },
            "required": [
              "price_usd"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "price_usd": null
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"price_usd\": null\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 44,
          "total_tokens": 61,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route-2",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write JavaScript that sorts an array of numbers."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"code\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 51,
          "total_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route-7",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Implement binary search in Python."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"Implement binary search in Python\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 21,
          "prompt_tokens": 48,
          "total_tokens": 69,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-2",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write JavaScript that sorts an array of numbers."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"code\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 51,
          "total_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-4",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: What does GPU stand for?"
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "lookup"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"Graphics Processing Unit\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 19,
          "prompt_tokens": 48,
          "total_tokens": 67,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-6",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request:    "
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "clarify"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"clarify\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 18,
          "prompt_tokens": 42,
          "total_tokens": 60,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write a Python function that reverses a list."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"code\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 51,
          "total_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473751
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route-4",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: What does GPU stand for?"
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "lookup"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"lookup\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 48,
          "total_tokens": 65,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "extract",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only JSON with model and memory_gb extracted from this text: I have an RTX 5090 with 32 GB VRAM."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "model": "RTX 5090",
        "memory_gb": 32
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"model\": \"RTX 5090\",\n  \"memory_gb\": 32\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 33,
          "prompt_tokens": 40,
          "total_tokens": 73,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-6",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request:    "
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "clarify"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"clarification\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 18,
          "prompt_tokens": 42,
          "total_tokens": 60,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route-7",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Implement binary search in Python."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "route": {
                "type": "string",
                "enum": [
                  "code",
                  "lookup",
                  "clarify"
                ]
              }
            },
            "required": [
              "route"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"route\": \"code\"\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 17,
          "prompt_tokens": 48,
          "total_tokens": 65,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    },
    {
      "case": "route-3",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Explain the difference between RAM and VRAM."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "lookup"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"programming_request\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 19,
          "prompt_tokens": 51,
          "total_tokens": 70,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "route",
      "condition": "prompt_only",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only a JSON object with the key route. Choose code for a programming request, lookup for a factual question, or clarify when no request is given. Request: Write a Python function that reverses a list."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        }
      },
      "expected": {
        "route": "code"
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "```json\n{\n  \"route\": \"Write a Python function that reverses a list.\"\n}\n```"
            }
          }
        ],
        "usage": {
          "completion_tokens": 24,
          "prompt_tokens": 51,
          "total_tokens": 75,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": false,
      "exact_value_pass": false
    },
    {
      "case": "extract",
      "condition": "schema",
      "request": {
        "model": "gemma-4-E2B-it-qat-q4_0",
        "messages": [
          {
            "role": "user",
            "content": "Return only JSON with model and memory_gb extracted from this text: I have an RTX 5090 with 32 GB VRAM."
          }
        ],
        "max_tokens": 96,
        "temperature": 0,
        "seed": 42,
        "stream": false,
        "cache_prompt": false,
        "chat_template_kwargs": {
          "enable_thinking": false
        },
        "response_format": {
          "type": "json_object",
          "schema": {
            "type": "object",
            "properties": {
              "model": {
                "type": "string"
              },
              "memory_gb": {
                "type": "integer"
              }
            },
            "required": [
              "model",
              "memory_gb"
            ],
            "additionalProperties": false
          }
        }
      },
      "expected": {
        "model": "RTX 5090",
        "memory_gb": 32
      },
      "response": {
        "choices": [
          {
            "finish_reason": "stop",
            "index": 0,
            "message": {
              "role": "assistant",
              "content": "{\n  \"model\": \"RTX 5090\",\n  \"memory_gb\": 32\n}\n"
            }
          }
        ],
        "usage": {
          "completion_tokens": 33,
          "prompt_tokens": 40,
          "total_tokens": 73,
          "prompt_tokens_details": {
            "cached_tokens": 0
          }
        },
        "created": 1789473752
      },
      "json_parse_pass": true,
      "exact_value_pass": true
    }
  ],
  "summary": {
    "prompt_only": {
      "replies": 9,
      "json_parse_pass": 0,
      "exact_value_pass": 0
    },
    "schema": {
      "replies": 9,
      "json_parse_pass": 9,
      "exact_value_pass": 8
    }
  }
}