{
 "dataset": "run1",
 "date": "2026-09-26",
 "engines": {
  "mlx-lm": {
   "agent_session_note": "the server's generation thread died with a Metal out-of-memory error (see its server.log): on turn 4 of the first session and on turn 3 after the restart",
   "concurrent4_tps": 81.66631875610831,
   "decode_runs": 3,
   "decode_tps": 25.057968056361943,
   "label": "mlx-lm 0.31.3 (mlx_lm.server)",
   "prompt_tokens_16k": 15492,
   "prompt_tokens_2k": 1993,
   "run": "run1",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 46.006882208021125,
   "ttft_2k_s": 5.869392166001489
  },
  "ollama": {
   "agent_after_restart_total_s": 80.91878983398783,
   "agent_after_restart_turn1_s": 77.19656850001775,
   "agent_after_restart_turns2_10_median_s": 0.44288254200364463,
   "agent_after_restart_turns_s": [
    77.19656850001775,
    0.45328362498548813,
    0.4464438330032863,
    0.435909082996659,
    0.34377637499710545,
    0.44288254200364463,
    0.3489950420043897,
    0.4448157919978257,
    0.3497532499895897,
    0.45636179199209437
   ],
   "agent_first_total_s": 81.01826495496789,
   "agent_first_turn1_s": 77.14216008299263,
   "agent_first_turns2_10_median_s": 0.4569756670098286,
   "agent_first_turns_s": [
    77.14216008299263,
    0.49299950001295656,
    0.4677947500022128,
    0.4743980829953216,
    0.3589850409771316,
    0.46205629099858925,
    0.35174729098798707,
    0.4569756670098286,
    0.3575853329966776,
    0.4535629159945529
   ],
   "agent_prompt_tokens_turn1": 22675,
   "concurrent4_tps": 21.948836782632895,
   "decode_runs": 3,
   "decode_tps": 22.65158492752519,
   "label": "Ollama 0.34.3 (qwen3.5:9b, Q4_K_M)",
   "prompt_tokens_16k": 15492,
   "prompt_tokens_2k": 1991,
   "run": "run1",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 49.89889954199316,
   "ttft_2k_s": 6.2788226249977015
  },
  "omlx": {
   "agent_after_restart_total_s": 5.079078624054091,
   "agent_after_restart_turn1_s": 0.5598607910214923,
   "agent_after_restart_turns2_10_median_s": 0.4403861250029877,
   "agent_after_restart_turns_s": [
    0.5598607910214923,
    0.4349640420114156,
    0.45842195898876525,
    0.6815554579952732,
    0.43158279100316577,
    0.4166357920039445,
    0.41570733301341534,
    0.4403861250029877,
    0.5759996249980759,
    0.6639647080155555
   ],
   "agent_first_total_s": 74.54253941692878,
   "agent_first_turn1_s": 67.4834337499924,
   "agent_first_turns2_10_median_s": 0.6832837919937447,
   "agent_first_turns_s": [
    67.4834337499924,
    1.6348022079910152,
    0.675471833004849,
    0.7403159999812488,
    0.662448624992976,
    0.7155865840031765,
    0.5761774579877965,
    0.6618723330029752,
    0.7091468339785933,
    0.6832837919937447
   ],
   "agent_prompt_tokens_turn1": 22675,
   "concurrent4_tps": 72.9123242950955,
   "decode_runs": 3,
   "decode_tps": 24.928716757633488,
   "label": "oMLX 0.7.0rc1 (SSD cache on)",
   "prompt_tokens_16k": 15494,
   "prompt_tokens_2k": 1992,
   "run": "run1",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 44.22852599999169,
   "ttft_2k_s": 5.819532000023173
  },
  "rapid-mlx": {
   "agent_after_restart_total_s": 123.96092670800863,
   "agent_after_restart_turn1_s": 68.95686129198293,
   "agent_after_restart_turns2_10_median_s": 7.426451958017424,
   "agent_after_restart_turns_s": [
    68.95686129198293,
    7.329591207992053,
    7.426451958017424,
    7.463186500011943,
    7.6710097920149565,
    7.06372000000556,
    7.865507499984233,
    7.956718875007937,
    1.435914458008483,
    0.7919651249831077
   ],
   "agent_first_total_s": 124.19053387604072,
   "agent_first_turn1_s": 69.03285925000091,
   "agent_first_turns2_10_median_s": 7.47857712500263,
   "agent_first_turns_s": [
    69.03285925000091,
    7.3921964170003776,
    7.502681000012672,
    7.47857712500263,
    7.69950416701613,
    7.112688666995382,
    7.852989125007298,
    7.90227995798341,
    1.4177651250211056,
    0.7989930420008022
   ],
   "agent_prompt_tokens_turn1": 22821,
   "label": "Rapid-MLX 0.15.2 (defaults, PFlash on)",
   "prompt_tokens_16k": 15493,
   "prompt_tokens_2k": 1990,
   "run": "run1",
   "ttft_16k_s": 8.60830491600791,
   "ttft_2k_s": 5.528374749992508
  },
  "rapid-mlx-pflash-off": {
   "agent_after_restart_total_s": 128.2445491249964,
   "agent_after_restart_turn1_s": 70.64322083300794,
   "agent_after_restart_turns2_10_median_s": 7.639065750001464,
   "agent_after_restart_turns_s": [
    70.64322083300794,
    7.639065750001464,
    7.647337999980664,
    7.634196875005728,
    7.933781584026292,
    7.238169958000071,
    8.260568583005806,
    8.830575125000905,
    1.587124124984257,
    0.8305082919832785
   ],
   "agent_first_total_s": 642.8936804579862,
   "agent_first_turn1_s": 71.42325545899803,
   "agent_first_turns2_10_median_s": 70.37846841602004,
   "agent_first_turns_s": [
    71.42325545899803,
    70.63649808298214,
    70.53475175000494,
    70.40554512501694,
    70.37846841602004,
    69.87526541698026,
    8.162104916002136,
    70.34210091698333,
    70.8521095830074,
    70.28358079199097
   ],
   "agent_prompt_tokens_turn1": 22821,
   "concurrent4_tps": 52.74088905533796,
   "decode_runs": 3,
   "decode_tps": 37.48594322978046,
   "label": "Rapid-MLX 0.15.2 (PFlash off)",
   "prompt_tokens_16k": 15494,
   "prompt_tokens_2k": 1995,
   "run": "run1",
   "toolcall_pass": 30,
   "toolcall_total": 30,
   "ttft_16k_s": 44.96910074999323,
   "ttft_2k_s": 5.515980083990144
  }
 },
 "machine": "MacBook Pro, Apple M3 Pro, 18 GB unified memory, macOS 15.6.1, on power",
 "machine_short": "an M3 Pro MacBook Pro with 18 GB",
 "model": "Qwen3.5-9B at 4-bit (MLX engines: mlx-community/Qwen3.5-9B-4bit @ 8b2b98c; Ollama: qwen3.5:9b, GGUF Q4_K_M, id 6488c96fa5fa)",
 "model_short": "Qwen3.5-9B at 4-bit",
 "runs": {
  "mlx-lm": "run1",
  "ollama": "run1",
  "omlx": "run1",
  "rapid-mlx": "run1",
  "rapid-mlx-pflash-off": "run1"
 },
 "source": "computed by landing/scripts/build_compare_pages.py from raw/*.json"
}