{
 "dataset": "run2",
 "date": "2026-09-26",
 "engines": {
  "mlx-lm": {
   "agent_after_restart_completed_turns_s": [
    68.85313783297897,
    0.8578828340105247
   ],
   "agent_after_restart_stopped_at_turn": 3,
   "agent_after_restart_turn1_s": 68.85313783297897,
   "agent_after_restart_turns_s": [
    68.85313783297897,
    0.8578828340105247,
    null
   ],
   "agent_first_completed_turns_s": [
    71.3846596670046,
    0.852325749990996,
    0.7687579579942394
   ],
   "agent_first_stopped_at_turn": 4,
   "agent_first_turn1_s": 71.3846596670046,
   "agent_first_turns_s": [
    71.3846596670046,
    0.852325749990996,
    0.7687579579942394,
    null
   ],
   "agent_session_note": "the server's generation thread died with a Metal out-of-memory error (see its server.log): on turn 4 of the first session and on turn 3 after the restart",
   "concurrent4_tps": 80.73633590487142,
   "decode_runs": 3,
   "decode_tps": 25.596399072542628,
   "label": "mlx-lm 0.31.3 (mlx_lm.server)",
   "prompt_tokens_16k": 15489,
   "prompt_tokens_2k": 1990,
   "run": "run2",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 45.59865849997732,
   "ttft_2k_s": 5.869889874971705
  },
  "ollama": {
   "agent_after_restart_total_s": 80.61953491700115,
   "agent_after_restart_turn1_s": 76.92682545800926,
   "agent_after_restart_turns2_10_median_s": 0.43869399998220615,
   "agent_after_restart_turns_s": [
    76.92682545800926,
    0.4611082500196062,
    0.43869399998220615,
    0.4424841669970192,
    0.3411620839906391,
    0.4329902080062311,
    0.3459608749835752,
    0.4397895000001881,
    0.351206874998752,
    0.4393135000136681
   ],
   "agent_first_total_s": 81.41007008202723,
   "agent_first_turn1_s": 77.58356112500769,
   "agent_first_turns2_10_median_s": 0.44357841700548306,
   "agent_first_turns_s": [
    77.58356112500769,
    0.482507708016783,
    0.4811634579964448,
    0.44603116597863846,
    0.34830012498423457,
    0.44357841700548306,
    0.34636795800179243,
    0.4423804590187501,
    0.3519441660027951,
    0.4842355000146199
   ],
   "agent_prompt_tokens_turn1": 22675,
   "concurrent4_tps": 22.462993276028516,
   "decode_runs": 3,
   "decode_tps": 22.81098541982159,
   "label": "Ollama 0.34.3 (qwen3.5:9b, Q4_K_M)",
   "prompt_tokens_16k": 15489,
   "prompt_tokens_2k": 1991,
   "run": "run2",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 49.92521020799177,
   "ttft_2k_s": 6.235937666991958
  },
  "omlx": {
   "agent_after_restart_total_s": 5.005675041000359,
   "agent_after_restart_turn1_s": 0.5636479170061648,
   "agent_after_restart_turns2_10_median_s": 0.43721295800060034,
   "agent_after_restart_turns_s": [
    0.5636479170061648,
    0.46273337499587797,
    0.43721295800060034,
    0.6495219999924302,
    0.41345266698044725,
    0.411626500019338,
    0.41877279098844156,
    0.41290716599905863,
    0.5638993330067024,
    0.6719003340112977
   ],
   "agent_first_total_s": 72.9848216669634,
   "agent_first_turn1_s": 66.9740024159837,
   "agent_first_turns2_10_median_s": 0.6572988330153748,
   "agent_first_turns_s": [
    66.9740024159837,
    0.8509200839907862,
    0.6588543329853565,
    0.6754250419908203,
    0.6226339589920826,
    0.6569644170231186,
    0.6177205829881132,
    0.6745918749948032,
    0.5964101249992382,
    0.6572988330153748
   ],
   "agent_prompt_tokens_turn1": 22675,
   "concurrent4_tps": 85.34429974149482,
   "decode_runs": 3,
   "decode_tps": 27.939471940237446,
   "label": "oMLX 0.7.0rc1 (SSD cache on)",
   "prompt_tokens_16k": 15484.5,
   "prompt_tokens_2k": 1989,
   "run": "run2",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_failed_runs": 1,
   "ttft_16k_s": 43.88157970851171,
   "ttft_2k_s": 5.732091916986974
  },
  "rapid-mlx": {
   "label": "Rapid-MLX 0.15.2 (defaults, PFlash on)",
   "prompt_tokens_16k": 15491,
   "prompt_tokens_2k": 1989,
   "run": "run2",
   "ttft_16k_s": 8.64106104199891,
   "ttft_2k_s": 5.554329750011675
  },
  "rapid-mlx-pflash-off": {
   "agent_after_restart_total_s": 695.8850691259431,
   "agent_after_restart_turn1_s": 69.10555154198664,
   "agent_after_restart_turns2_10_median_s": 69.55218337499537,
   "agent_after_restart_turns_s": [
    69.10555154198664,
    68.77873104199534,
    69.05500520800706,
    69.55218337499537,
    69.29312870898866,
    69.13568370899884,
    69.76470158298616,
    70.09369416598929,
    71.12781999999424,
    69.97856979200151
   ],
   "agent_first_total_s": 702.2549238349893,
   "agent_first_turn1_s": 69.8670762499969,
   "agent_first_turns2_10_median_s": 69.98064216598868,
   "agent_first_turns_s": [
    69.8670762499969,
    69.65575020801043,
    69.53033687500283,
    69.6298346669937,
    69.98064216598868,
    69.68102270900272,
    70.70220295898616,
    71.64042991699534,
    70.82991141700768,
    70.73771666700486
   ],
   "agent_prompt_tokens_turn1": 22819,
   "concurrent4_tps": 60.8814449953983,
   "decode_runs": 3,
   "decode_tps": 40.856861533595726,
   "label": "Rapid-MLX 0.15.2 (PFlash off)",
   "prompt_tokens_16k": 15488,
   "prompt_tokens_2k": 1995,
   "run": "run2",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 45.663971084024524,
   "ttft_2k_s": 5.587942374986596
  }
 },
 "machine": "MacBook Pro, Apple M3 Pro, 18 GB unified memory, macOS 15.6.1, on power",
 "machine_short": "an M3 Pro MacBook Pro with 18 GB",
 "model": "Qwen3.5-9B at 4-bit (MLX engines: mlx-community/Qwen3.5-9B-4bit @ 8b2b98c; Ollama: qwen3.5:9b, GGUF Q4_K_M, id 6488c96fa5fa)",
 "model_short": "Qwen3.5-9B at 4-bit",
 "runs": {
  "mlx-lm": "run2",
  "ollama": "run2",
  "omlx": "run2",
  "rapid-mlx": "run2",
  "rapid-mlx-pflash-off": "run2"
 },
 "source": "computed by landing/scripts/build_compare_pages.py from raw/*.json"
}