{
 "claims": {
  "headline_model": "9b",
  "headline_x": 4,
  "median_x": 1.5
 },
 "date": "2026-10-04",
 "extras": {
  "mlxlm": {
   "agent_after_restart_turn1_s": 65.58772695800872,
   "agent_prompt_tokens_turn1": 22675,
   "agent_turn1_s": 65.60963462499785,
   "agent_turns2_10_median_s": 0.35798841700307094,
   "concurrent4_reps": 2,
   "concurrent4_tps": 100.16723694497895
  },
  "rapid": {
   "agent_after_restart_turn1_s": 0.34023066598456353,
   "agent_prompt_tokens_turn1": 22819,
   "agent_turn1_s": 65.6901382500073,
   "agent_turns2_10_median_s": 0.468166457983898,
   "concurrent4_reps": 2,
   "concurrent4_tps": 104.15557658719263
  }
 },
 "generate": {
  "35b": {
   "code_rename": {
    "tokens": 1408,
    "tps": 87.244
   },
   "long_answer": {
    "tokens": 1049,
    "tps": 90.228
   }
  },
  "4b": {
   "code_rename": {
    "tokens": 1408,
    "tps": 82.175
   },
   "long_answer": {
    "tokens": 1029,
    "tps": 83.842
   }
  },
  "9b": {
   "code_rename": {
    "tokens": 1408,
    "tps": 49.576
   },
   "long_answer": {
    "tokens": 1100,
    "tps": 50.264
   }
  }
 },
 "machine": "Mac mini, Apple M4 Pro, 48 GB unified memory, macOS 26.5.1, on AC power, idle before every server start (30 s rest, then 10 seconds of idle GPU; no other model server running)",
 "machine_inline": "Mac mini M4 Pro (48 GB)",
 "machine_short": "Mac mini M4 Pro, 48 GB",
 "measured": "Oct 4, 2026",
 "mlxlm_version": "0.32.0",
 "rapid_version": "0.15.6",
 "sets": {
  "35b": {
   "best": "code_rename",
   "best_vs_generate_x": 2.6085432133222595,
   "config": "rapid",
   "decode": {
    "geomean": 1.6960403391049232,
    "max": 3.163458726927386,
    "median": 1.5682120691358894,
    "min": 1.1906683459889122,
    "n": 18
   },
   "e2e": {
    "geomean": 1.5126829758307312,
    "max": 2.658433955406295,
    "median": 1.4185740723238478,
    "min": 1.0965252473691367,
    "n": 20
   },
   "generate_tps": 87.244,
   "identical": 9,
   "label": "Qwen3.6-35B-A3B 4-bit (MoE)",
   "mlx_reps": 3,
   "model": "35b",
   "reps": 3,
   "rows": [
    {
     "decode_x": 3.163458726927386,
     "e2e_x": 2.658433955406295,
     "first_diff_char": null,
     "id": "code_rename",
     "identical": true,
     "mlx_e2e_s": 21.44190345800598,
     "mlx_tokens": 1408,
     "mlx_tps": 71.94016541639269,
     "mlx_ttft_s": 1.8374463749933057,
     "rapid_e2e_s": 8.065614499995718,
     "rapid_tokens": 1408,
     "rapid_tps": 227.5797441030872,
     "rapid_ttft_s": 1.8172948749852367,
     "task": "Code edit: rename across a file"
    },
    {
     "decode_x": 3.094011551892764,
     "e2e_x": 2.6435899146540023,
     "first_diff_char": 3932,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 28.57399391598301,
     "mlx_tokens": 1859,
     "mlx_tps": 71.28624410131299,
     "mlx_ttft_s": 2.450946540979203,
     "rapid_e2e_s": 10.808784584020032,
     "rapid_tokens": 1877,
     "rapid_tps": 220.5604627405098,
     "rapid_ttft_s": 2.306101124995621,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 2.786889327612278,
     "e2e_x": 2.4492463101374,
     "first_diff_char": null,
     "id": "apply_change_request",
     "identical": true,
     "mlx_e2e_s": 35.19208520799293,
     "mlx_tokens": 2261,
     "mlx_tps": 70.58956459007017,
     "mlx_ttft_s": 3.1165920829807874,
     "rapid_e2e_s": 14.368536583002424,
     "rapid_tokens": 2261,
     "rapid_tps": 196.72530419686413,
     "rapid_ttft_s": 2.8912286249978933,
     "task": "Code edit: apply review comments"
    },
    {
     "decode_x": 1.4666145364550272,
     "e2e_x": 1.309854721187886,
     "first_diff_char": 64,
     "id": "unified_diff",
     "identical": false,
     "mlx_e2e_s": 5.407941790996119,
     "mlx_tokens": 221,
     "mlx_tps": 72.33174759509551,
     "mlx_ttft_s": 2.35460312498617,
     "rapid_e2e_s": 4.12865770800272,
     "rapid_tokens": 207,
     "rapid_tps": 106.08279247016303,
     "rapid_ttft_s": 2.165840957983164,
     "task": "Code edit: produce a unified diff"
    },
    {
     "decode_x": 1.6031421889724338,
     "e2e_x": 1.5307555906443668,
     "first_diff_char": null,
     "id": "json_transform",
     "identical": true,
     "mlx_e2e_s": 21.487199833005434,
     "mlx_tokens": 1395,
     "mlx_tps": 71.57202797585599,
     "mlx_ttft_s": 1.9560654589731712,
     "rapid_e2e_s": 14.036989290994825,
     "rapid_tokens": 1395,
     "rapid_tps": 114.74013759841003,
     "rapid_ttft_s": 1.886874041025294,
     "task": "JSON transform (rename keys, convert types)"
    },
    {
     "decode_x": 1.5623361403516693,
     "e2e_x": 1.4207340858216462,
     "first_diff_char": null,
     "id": "json_to_csv",
     "identical": true,
     "mlx_e2e_s": 9.363619707990438,
     "mlx_tokens": 532,
     "mlx_tps": 72.39454206754775,
     "mlx_ttft_s": 2.006247915996937,
     "rapid_e2e_s": 6.590691249992233,
     "rapid_tokens": 532,
     "rapid_tps": 113.1046094363391,
     "rapid_ttft_s": 1.878865249978844,
     "task": "Format conversion: JSON to CSV"
    },
    {
     "decode_x": 1.5669181157157863,
     "e2e_x": 1.4162627525726608,
     "first_diff_char": null,
     "id": "json_schema_fill",
     "identical": true,
     "mlx_e2e_s": 4.801019083010033,
     "mlx_tokens": 278,
     "mlx_tps": 74.00815626895219,
     "mlx_ttft_s": 1.0417275410145521,
     "rapid_e2e_s": 3.3899211670213845,
     "rapid_tokens": 278,
     "rapid_tps": 115.96472076854603,
     "rapid_ttft_s": 1.0008038750092965,
     "task": "Structured output: fill a JSON schema"
    },
    {
     "decode_x": 1.6141924434188373,
     "e2e_x": 1.5022881860445394,
     "first_diff_char": null,
     "id": "extract_fields",
     "identical": true,
     "mlx_e2e_s": 5.731234499980928,
     "mlx_tokens": 373,
     "mlx_tps": 74.12461777938191,
     "mlx_ttft_s": 0.690670041978592,
     "rapid_e2e_s": 3.8150033749989234,
     "rapid_tokens": 373,
     "rapid_tps": 119.65139789078786,
     "rapid_ttft_s": 0.686998165998375,
     "task": "Extract fields from a document"
    },
    {
     "decode_x": null,
     "e2e_x": 1.2254852324789585,
     "first_diff_char": null,
     "id": "tool_call",
     "identical": true,
     "mlx_e2e_s": 3.076881582994247,
     "mlx_tokens": 148,
     "mlx_tps": null,
     "mlx_ttft_s": 3.064065999991726,
     "rapid_e2e_s": 2.5107455409888644,
     "rapid_tokens": 148,
     "rapid_tps": 118.94703155148012,
     "rapid_ttft_s": 1.2859127080009785,
     "task": "Tool call (OpenAI tools)"
    },
    {
     "decode_x": null,
     "e2e_x": 1.0965252473691367,
     "first_diff_char": null,
     "id": "agent_step_8k",
     "identical": true,
     "mlx_e2e_s": 15.168628541985527,
     "mlx_tokens": 214,
     "mlx_tps": null,
     "mlx_ttft_s": 15.15205895798863,
     "rapid_e2e_s": 13.833360041986452,
     "rapid_tokens": 214,
     "rapid_tps": 105.31667719064961,
     "rapid_ttft_s": 11.853444666980067,
     "task": "Coding-agent turn, ~8k-token context with tool results"
    },
    {
     "decode_x": 1.364082759927038,
     "e2e_x": 1.2533254339750823,
     "first_diff_char": 317,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 5.360014999983832,
     "mlx_tokens": 316,
     "mlx_tps": 73.76896143424504,
     "mlx_ttft_s": 1.075756124977488,
     "rapid_e2e_s": 4.276634667010512,
     "rapid_tokens": 319,
     "rapid_tps": 100.6269685101762,
     "rapid_ttft_s": 1.0949657500023022,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.5556633391502048,
     "e2e_x": 1.4164140588260496,
     "first_diff_char": 197,
     "id": "translate",
     "identical": false,
     "mlx_e2e_s": 2.9149421250040177,
     "mlx_tokens": 190,
     "mlx_tps": 74.81958187192689,
     "mlx_ttft_s": 0.37485154200112447,
     "rapid_e2e_s": 2.0579731659963727,
     "rapid_tokens": 189,
     "rapid_tps": 116.39408056870391,
     "rapid_ttft_s": 0.4149409999954514,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 1.9591422333410582,
     "e2e_x": 1.7620406508701338,
     "first_diff_char": null,
     "id": "proofread_doc",
     "identical": true,
     "mlx_e2e_s": 6.6052892500010785,
     "mlx_tokens": 436,
     "mlx_tps": 74.22845915698498,
     "mlx_ttft_s": 0.7253277499985415,
     "rapid_e2e_s": 3.7486588330066297,
     "rapid_tokens": 436,
     "rapid_tps": 145.42410925028108,
     "rapid_ttft_s": 0.7560294580180198,
     "task": "Proofread a document, return full text"
    },
    {
     "decode_x": 1.4313225653256152,
     "e2e_x": 1.4014423675987746,
     "first_diff_char": 300,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 2.28393716699793,
     "mlx_tokens": 154,
     "mlx_tps": 75.05754871974091,
     "mlx_ttft_s": 0.2327285410137847,
     "rapid_e2e_s": 1.629704666993348,
     "rapid_tokens": 148,
     "rapid_tps": 107.4315631805919,
     "rapid_ttft_s": 0.2533095830003731,
     "task": "Write an email"
    },
    {
     "decode_x": 1.1906683459889122,
     "e2e_x": 1.120300467174049,
     "first_diff_char": 461,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 8.429286166996462,
     "mlx_tokens": 612,
     "mlx_tps": 74.28432907481087,
     "mlx_ttft_s": 0.1802149169961922,
     "rapid_e2e_s": 7.524129833007464,
     "rapid_tokens": 648,
     "rapid_tps": 88.44799923240112,
     "rapid_ttft_s": 0.1964586660033092,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.2909658919637126,
     "e2e_x": 1.2257913116459969,
     "first_diff_char": 124,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 11.963279209012398,
     "mlx_tokens": 700,
     "mlx_tps": 71.72933351370177,
     "mlx_ttft_s": 2.2182794170221314,
     "rapid_e2e_s": 9.75963779099402,
     "rapid_tokens": 700,
     "rapid_tps": 92.60012301947863,
     "rapid_ttft_s": 2.205734624993056,
     "task": "Explain code"
    },
    {
     "decode_x": 1.5751041522361389,
     "e2e_x": 1.4019103109050204,
     "first_diff_char": 14,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 4.509612166002626,
     "mlx_tokens": 306,
     "mlx_tps": 74.53299048524313,
     "mlx_ttft_s": 0.4007969159865752,
     "rapid_e2e_s": 3.2167622499982826,
     "rapid_tokens": 328,
     "rapid_tps": 117.39722279188308,
     "rapid_ttft_s": 0.41394870801013894,
     "task": "SQL"
    },
    {
     "decode_x": 1.5695060225559923,
     "e2e_x": 1.5276804515106694,
     "first_diff_char": 504,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 6.316413666994777,
     "mlx_tokens": 450,
     "mlx_tps": 74.38291995881539,
     "mlx_ttft_s": 0.27975412501837127,
     "rapid_e2e_s": 4.134643249999499,
     "rapid_tokens": 450,
     "rapid_tps": 116.74444085066108,
     "rapid_ttft_s": 0.288516582979355,
     "task": "Regex"
    },
    {
     "decode_x": 1.5901201421790172,
     "e2e_x": 1.4838910252513304,
     "first_diff_char": 495,
     "id": "markdown_table",
     "identical": false,
     "mlx_e2e_s": 4.375506000011228,
     "mlx_tokens": 289,
     "mlx_tps": 74.47590250400876,
     "mlx_ttft_s": 0.4920364169811364,
     "rapid_e2e_s": 2.948670707992278,
     "rapid_tokens": 287,
     "rapid_tps": 118.42563267858503,
     "rapid_ttft_s": 0.5171656249731313,
     "task": "Markdown table from notes"
    },
    {
     "decode_x": 1.4798430197929964,
     "e2e_x": 1.467199972130147,
     "first_diff_char": 404,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 13.815279832982924,
     "mlx_tokens": 1006,
     "mlx_tps": 74.12754393384786,
     "mlx_ttft_s": 0.22382937499787658,
     "rapid_e2e_s": 9.416085125005338,
     "rapid_tokens": 1007,
     "rapid_tps": 109.69712846490343,
     "rapid_ttft_s": 0.24609258302371018,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 20
  },
  "4b": {
   "best": "apply_change_request",
   "config": "rapid",
   "decode": {
    "geomean": 1.1191780858560465,
    "max": 1.1410300748794282,
    "median": 1.1171446823905735,
    "min": 1.1112052344821202,
    "n": 18
   },
   "e2e": {
    "geomean": 1.1033915916331876,
    "max": 1.2390258864464319,
    "median": 1.1043177837697375,
    "min": 0.9935289482598001,
    "n": 20
   },
   "identical": 9,
   "label": "Qwen3.5-4B 4-bit",
   "mlx_reps": 3,
   "model": "4b",
   "reps": 3,
   "rows": [
    {
     "decode_x": 1.1262918379618692,
     "e2e_x": 1.115550862209081,
     "first_diff_char": null,
     "id": "code_rename",
     "identical": true,
     "mlx_e2e_s": 21.4482051669911,
     "mlx_tokens": 1408,
     "mlx_tps": 73.14545882340778,
     "mlx_ttft_s": 2.177372416976141,
     "rapid_e2e_s": 19.226559625007212,
     "rapid_tokens": 1408,
     "rapid_tps": 82.38313325678017,
     "rapid_ttft_s": 2.135693624994019,
     "task": "Code edit: rename across a file"
    },
    {
     "decode_x": 1.1345427030504307,
     "e2e_x": 1.1361793886543274,
     "first_diff_char": 4090,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 29.511459750006907,
     "mlx_tokens": 1921,
     "mlx_tps": 72.26259515744228,
     "mlx_ttft_s": 2.901122375013074,
     "rapid_e2e_s": 25.974296000000322,
     "rapid_tokens": 1900,
     "rapid_tps": 81.98500003936353,
     "rapid_ttft_s": 2.7984760830004234,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 1.1410300748794282,
     "e2e_x": 1.133004366037209,
     "first_diff_char": null,
     "id": "apply_change_request",
     "identical": true,
     "mlx_e2e_s": 35.47190170799149,
     "mlx_tokens": 2261,
     "mlx_tps": 71.14683012965178,
     "mlx_ttft_s": 3.65730895800516,
     "rapid_e2e_s": 31.30782437499147,
     "rapid_tokens": 2261,
     "rapid_tps": 81.18067291027052,
     "rapid_ttft_s": 3.45144916701247,
     "task": "Code edit: apply review comments"
    },
    {
     "decode_x": 1.1230936430971672,
     "e2e_x": 1.1146321975176496,
     "first_diff_char": null,
     "id": "unified_diff",
     "identical": true,
     "mlx_e2e_s": 15.467964874987956,
     "mlx_tokens": 900,
     "mlx_tps": 72.14976211799703,
     "mlx_ttft_s": 3.007734790997347,
     "rapid_e2e_s": 13.877191874984419,
     "rapid_tokens": 900,
     "rapid_tps": 81.03093918569527,
     "rapid_ttft_s": 2.7824841669935267,
     "task": "Code edit: produce a unified diff"
    },
    {
     "decode_x": 1.117948179755469,
     "e2e_x": 1.1080247326509864,
     "first_diff_char": null,
     "id": "json_transform",
     "identical": true,
     "mlx_e2e_s": 21.996189958998002,
     "mlx_tokens": 1408,
     "mlx_tps": 72.37482629582857,
     "mlx_ttft_s": 2.536323292006273,
     "rapid_e2e_s": 19.85171387498849,
     "rapid_tokens": 1408,
     "rapid_tps": 80.9113053175398,
     "rapid_ttft_s": 2.447973666974576,
     "task": "JSON transform (rename keys, convert types)"
    },
    {
     "decode_x": 1.1175640839245788,
     "e2e_x": 1.0945132220113976,
     "first_diff_char": null,
     "id": "json_to_csv",
     "identical": true,
     "mlx_e2e_s": 9.808214499993483,
     "mlx_tokens": 537,
     "mlx_tps": 73.01107023461206,
     "mlx_ttft_s": 2.4501915409928188,
     "rapid_e2e_s": 8.961257207993185,
     "rapid_tokens": 537,
     "rapid_tps": 81.59454982309731,
     "rapid_ttft_s": 2.378099374996964,
     "task": "Format conversion: JSON to CSV"
    },
    {
     "decode_x": 1.114684985249539,
     "e2e_x": 1.087017837986989,
     "first_diff_char": null,
     "id": "json_schema_fill",
     "identical": true,
     "mlx_e2e_s": 4.899090250022709,
     "mlx_tokens": 274,
     "mlx_tps": 74.52377719661813,
     "mlx_ttft_s": 1.223010208021151,
     "rapid_e2e_s": 4.5069087910233065,
     "rapid_tokens": 274,
     "rapid_tps": 83.07053548515222,
     "rapid_ttft_s": 1.206656958005624,
     "task": "Structured output: fill a JSON schema"
    },
    {
     "decode_x": 1.11574286449918,
     "e2e_x": 1.0995867262681298,
     "first_diff_char": null,
     "id": "extract_fields",
     "identical": true,
     "mlx_e2e_s": 5.870079000014812,
     "mlx_tokens": 381,
     "mlx_tps": 74.65571498842287,
     "mlx_ttft_s": 0.7670163750008214,
     "rapid_e2e_s": 5.338441125000827,
     "rapid_tokens": 381,
     "rapid_tps": 83.2965812924173,
     "rapid_ttft_s": 0.7683917920221575,
     "task": "Extract fields from a document"
    },
    {
     "decode_x": null,
     "e2e_x": 1.2390258864464319,
     "first_diff_char": 1,
     "id": "tool_call",
     "identical": false,
     "mlx_e2e_s": 3.3574690840032417,
     "mlx_tokens": 166,
     "mlx_tps": 75.14285541413304,
     "mlx_ttft_s": 1.14903158400557,
     "rapid_e2e_s": 2.709765082981903,
     "rapid_tokens": 113,
     "rapid_tps": null,
     "rapid_ttft_s": 2.6960827499860898,
     "task": "Tool call (OpenAI tools)"
    },
    {
     "decode_x": null,
     "e2e_x": 0.9935289482598001,
     "first_diff_char": 1,
     "id": "agent_step_8k",
     "identical": false,
     "mlx_e2e_s": 16.970744207996177,
     "mlx_tokens": 217,
     "mlx_tps": null,
     "mlx_ttft_s": 16.956504582980415,
     "rapid_e2e_s": 17.081278041994665,
     "rapid_tokens": 235,
     "rapid_tps": 72.02960600799237,
     "rapid_ttft_s": 13.817441791994497,
     "task": "Coding-agent turn, ~8k-token context with tool results"
    },
    {
     "decode_x": 1.1143678603090423,
     "e2e_x": 1.1408049976433752,
     "first_diff_char": 31,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 4.537891125015449,
     "mlx_tokens": 248,
     "mlx_tps": 74.73671572772957,
     "mlx_ttft_s": 1.2202457500097807,
     "rapid_e2e_s": 3.9777973750024103,
     "rapid_tokens": 230,
     "rapid_tps": 83.28419399203516,
     "rapid_ttft_s": 1.2146340830076952,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.1112052344821202,
     "e2e_x": 1.0932843966719117,
     "first_diff_char": null,
     "id": "translate",
     "identical": true,
     "mlx_e2e_s": 2.7637809999869205,
     "mlx_tokens": 184,
     "mlx_tps": 75.76583657328679,
     "mlx_ttft_s": 0.3357772089948412,
     "rapid_e2e_s": 2.5279616249899846,
     "rapid_tokens": 184,
     "rapid_tps": 84.19139419515314,
     "rapid_ttft_s": 0.34112304099835455,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 1.1172206770080182,
     "e2e_x": 1.1006108348884889,
     "first_diff_char": null,
     "id": "proofread_doc",
     "identical": true,
     "mlx_e2e_s": 6.631486250000307,
     "mlx_tokens": 437,
     "mlx_tps": 74.7661131423701,
     "mlx_ttft_s": 0.7848407080164179,
     "rapid_e2e_s": 6.025277999986429,
     "rapid_tokens": 437,
     "rapid_tps": 83.53024754217681,
     "rapid_ttft_s": 0.7876854999922216,
     "task": "Proofread a document, return full text"
    },
    {
     "decode_x": 1.1123337533189754,
     "e2e_x": 1.0370055537204466,
     "first_diff_char": 513,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 1.7737174160138238,
     "mlx_tokens": 123,
     "mlx_tps": 76.24833406856452,
     "mlx_ttft_s": 0.1611188749957364,
     "rapid_e2e_s": 1.7104222920024768,
     "rapid_tokens": 127,
     "rapid_tps": 84.81359561880548,
     "rapid_ttft_s": 0.2091372919967398,
     "task": "Write an email"
    },
    {
     "decode_x": 1.117068687773129,
     "e2e_x": 1.0490738016393466,
     "first_diff_char": 5,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 8.692362333997153,
     "mlx_tokens": 641,
     "mlx_tps": 75.03538167668695,
     "mlx_ttft_s": 0.14460258299368434,
     "rapid_e2e_s": 8.285749125003349,
     "rapid_tokens": 681,
     "rapid_tps": 83.81967534613258,
     "rapid_ttft_s": 0.15980458399280906,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.11780311205643,
     "e2e_x": 1.0948455460310074,
     "first_diff_char": 146,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 12.20140445799916,
     "mlx_tokens": 700,
     "mlx_tps": 73.04723399165533,
     "mlx_ttft_s": 2.632221957988804,
     "rapid_e2e_s": 11.14440708301845,
     "rapid_tokens": 700,
     "rapid_tps": 81.65242548298657,
     "rapid_ttft_s": 2.5784102079924196,
     "task": "Explain code"
    },
    {
     "decode_x": 1.115334359432859,
     "e2e_x": 1.130345303211671,
     "first_diff_char": 1224,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 4.981585666973842,
     "mlx_tokens": 349,
     "mlx_tps": 75.42138380675073,
     "mlx_ttft_s": 0.3544758750067558,
     "rapid_e2e_s": 4.407136166992132,
     "rapid_tokens": 342,
     "rapid_tps": 84.12006079564213,
     "rapid_ttft_s": 0.34032925000065006,
     "task": "SQL"
    },
    {
     "decode_x": 1.1157346873069935,
     "e2e_x": 1.1106532007400185,
     "first_diff_char": 80,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 6.167855625011725,
     "mlx_tokens": 450,
     "mlx_tps": 75.24481378624442,
     "mlx_ttft_s": 0.20061483400058933,
     "rapid_e2e_s": 5.553358709003078,
     "rapid_tokens": 450,
     "rapid_tps": 83.95324878126837,
     "rapid_ttft_s": 0.20556666600168683,
     "task": "Regex"
    },
    {
     "decode_x": 1.1150007679631124,
     "e2e_x": 1.0964592517664014,
     "first_diff_char": 603,
     "id": "markdown_table",
     "identical": false,
     "mlx_e2e_s": 4.466338542028097,
     "mlx_tokens": 301,
     "mlx_tps": 75.47583979050702,
     "mlx_ttft_s": 0.47885933300131,
     "rapid_e2e_s": 4.07341954097501,
     "rapid_tokens": 303,
     "rapid_tps": 84.15561932907617,
     "rapid_ttft_s": 0.4719483329972718,
     "task": "Markdown table from notes"
    },
    {
     "decode_x": 1.1186829650012386,
     "e2e_x": 1.1130021114945141,
     "first_diff_char": 1627,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 14.862625249981647,
     "mlx_tokens": 1100,
     "mlx_tps": 74.73109280188136,
     "mlx_ttft_s": 0.15661370800808072,
     "rapid_e2e_s": 13.353636166983051,
     "rapid_tokens": 1100,
     "rapid_tps": 83.60040047339136,
     "rapid_ttft_s": 0.20739691698690876,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 20
  },
  "4b_mtp": {
   "best": "code_rename",
   "config": "rapid_mtp",
   "decode": {
    "geomean": 1.7629431978459333,
    "max": 4.1285607628914915,
    "median": 1.5249518515962444,
    "min": 1.2759790297606477,
    "n": 18
   },
   "e2e": {
    "geomean": 1.5624968343977599,
    "max": 3.202874320850376,
    "median": 1.4224930806263085,
    "min": 1.0471785815037677,
    "n": 20
   },
   "identical": 10,
   "label": "Qwen3.5-4B 4-bit",
   "mlx_reps": 3,
   "model": "4b",
   "reps": 3,
   "rows": [
    {
     "decode_x": 4.1285607628914915,
     "e2e_x": 3.1567874886296274,
     "first_diff_char": null,
     "id": "code_rename",
     "identical": true,
     "mlx_e2e_s": 21.4482051669911,
     "mlx_tokens": 1408,
     "mlx_tps": 73.14545882340778,
     "mlx_ttft_s": 2.177372416976141,
     "rapid_e2e_s": 6.794313916994724,
     "rapid_tokens": 1408,
     "rapid_tps": 301.9854712820166,
     "rapid_ttft_s": 2.128068334015552,
     "task": "Code edit: rename across a file"
    },
    {
     "decode_x": 4.048329588876277,
     "e2e_x": 3.202874320850376,
     "first_diff_char": 4090,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 29.511459750006907,
     "mlx_tokens": 1921,
     "mlx_tps": 72.26259515744228,
     "mlx_ttft_s": 2.901122375013074,
     "rapid_e2e_s": 9.214054874988506,
     "rapid_tokens": 1885,
     "rapid_tps": 292.5428021448612,
     "rapid_ttft_s": 2.7749422079941723,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 3.7200072389822942,
     "e2e_x": 2.9631025142594423,
     "first_diff_char": null,
     "id": "apply_change_request",
     "identical": true,
     "mlx_e2e_s": 35.47190170799149,
     "mlx_tokens": 2261,
     "mlx_tps": 71.14683012965178,
     "mlx_ttft_s": 3.65730895800516,
     "rapid_e2e_s": 11.971202999993693,
     "rapid_tokens": 2261,
     "rapid_tps": 264.6667231129482,
     "rapid_ttft_s": 3.414576249982929,
     "task": "Code edit: apply review comments"
    },
    {
     "decode_x": 1.4867883993966353,
     "e2e_x": 1.4100506936803394,
     "first_diff_char": null,
     "id": "unified_diff",
     "identical": true,
     "mlx_e2e_s": 15.467964874987956,
     "mlx_tokens": 900,
     "mlx_tps": 72.14976211799703,
     "mlx_ttft_s": 3.007734790997347,
     "rapid_e2e_s": 10.96979345800355,
     "rapid_tokens": 900,
     "rapid_tps": 107.27142933626479,
     "rapid_ttft_s": 2.588960916007636,
     "task": "Code edit: produce a unified diff"
    },
    {
     "decode_x": 1.6037515097203323,
     "e2e_x": 1.5292307752836272,
     "first_diff_char": null,
     "id": "json_transform",
     "identical": true,
     "mlx_e2e_s": 21.996189958998002,
     "mlx_tokens": 1408,
     "mlx_tps": 72.37482629582857,
     "mlx_ttft_s": 2.536323292006273,
     "rapid_e2e_s": 14.38382637500763,
     "rapid_tokens": 1408,
     "rapid_tps": 116.07123693768187,
     "rapid_ttft_s": 2.263239416992292,
     "task": "JSON transform (rename keys, convert types)"
    },
    {
     "decode_x": 1.530621732786026,
     "e2e_x": 1.3937479676242752,
     "first_diff_char": null,
     "id": "json_to_csv",
     "identical": true,
     "mlx_e2e_s": 9.808214499993483,
     "mlx_tokens": 537,
     "mlx_tps": 73.01107023461206,
     "mlx_ttft_s": 2.4501915409928188,
     "rapid_e2e_s": 7.037294208013918,
     "rapid_tokens": 537,
     "rapid_tps": 111.75233083506417,
     "rapid_ttft_s": 2.223413875006372,
     "task": "Format conversion: JSON to CSV"
    },
    {
     "decode_x": 1.58264911467326,
     "e2e_x": 1.4089206150778082,
     "first_diff_char": null,
     "id": "json_schema_fill",
     "identical": true,
     "mlx_e2e_s": 4.899090250022709,
     "mlx_tokens": 274,
     "mlx_tps": 74.52377719661813,
     "mlx_ttft_s": 1.223010208021151,
     "rapid_e2e_s": 3.4771939579804894,
     "rapid_tokens": 274,
     "rapid_tps": 117.94499000233496,
     "rapid_ttft_s": 1.1620249579718802,
     "task": "Structured output: fill a JSON schema"
    },
    {
     "decode_x": 1.6020530985947279,
     "e2e_x": 1.4881119798115003,
     "first_diff_char": null,
     "id": "extract_fields",
     "identical": true,
     "mlx_e2e_s": 5.870079000014812,
     "mlx_tokens": 381,
     "mlx_tps": 74.65571498842287,
     "mlx_ttft_s": 0.7670163750008214,
     "rapid_e2e_s": 3.9446487090026494,
     "rapid_tokens": 381,
     "rapid_tps": 119.60241952500773,
     "rapid_ttft_s": 0.7500948749948293,
     "task": "Extract fields from a document"
    },
    {
     "decode_x": null,
     "e2e_x": 1.4224538726803757,
     "first_diff_char": 1,
     "id": "tool_call",
     "identical": false,
     "mlx_e2e_s": 3.3574690840032417,
     "mlx_tokens": 166,
     "mlx_tps": 75.14285541413304,
     "mlx_ttft_s": 1.14903158400557,
     "rapid_e2e_s": 2.360335999983363,
     "rapid_tokens": 113,
     "rapid_tps": null,
     "rapid_ttft_s": 2.3421888329903595,
     "task": "Tool call (OpenAI tools)"
    },
    {
     "decode_x": null,
     "e2e_x": 1.0471785815037677,
     "first_diff_char": 1,
     "id": "agent_step_8k",
     "identical": false,
     "mlx_e2e_s": 16.970744207996177,
     "mlx_tokens": 217,
     "mlx_tps": null,
     "mlx_ttft_s": 16.956504582980415,
     "rapid_e2e_s": 16.20616054200218,
     "rapid_tokens": 235,
     "rapid_tps": 93.37467600037562,
     "rapid_ttft_s": 13.59920945801423,
     "task": "Coding-agent turn, ~8k-token context with tool results"
    },
    {
     "decode_x": 1.3728225730517358,
     "e2e_x": 1.2373777275699294,
     "first_diff_char": 478,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 4.537891125015449,
     "mlx_tokens": 248,
     "mlx_tps": 74.73671572772957,
     "mlx_ttft_s": 1.2202457500097807,
     "rapid_e2e_s": 3.6673450829985086,
     "rapid_tokens": 247,
     "rapid_tps": 102.60025038677784,
     "rapid_ttft_s": 1.2041069580009207,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.5192819704064628,
     "e2e_x": 1.4212167513859446,
     "first_diff_char": null,
     "id": "translate",
     "identical": true,
     "mlx_e2e_s": 2.7637809999869205,
     "mlx_tokens": 184,
     "mlx_tps": 75.76583657328679,
     "mlx_ttft_s": 0.3357772089948412,
     "rapid_e2e_s": 1.9446583339886274,
     "rapid_tokens": 184,
     "rapid_tps": 115.10966947855721,
     "rapid_ttft_s": 0.3378049999882933,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 2.080292913243948,
     "e2e_x": 1.8443089078250494,
     "first_diff_char": null,
     "id": "proofread_doc",
     "identical": true,
     "mlx_e2e_s": 6.631486250000307,
     "mlx_tokens": 437,
     "mlx_tps": 74.7661131423701,
     "mlx_ttft_s": 0.7848407080164179,
     "rapid_e2e_s": 3.5956483330228366,
     "rapid_tokens": 437,
     "rapid_tps": 155.53541532086774,
     "rapid_ttft_s": 0.7949089999892749,
     "task": "Proofread a document, return full text"
    },
    {
     "decode_x": 1.2759790297606477,
     "e2e_x": 1.2026463903241902,
     "first_diff_char": 513,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 1.7737174160138238,
     "mlx_tokens": 123,
     "mlx_tps": 76.24833406856452,
     "mlx_ttft_s": 0.1611188749957364,
     "rapid_e2e_s": 1.4748453329957556,
     "rapid_tokens": 123,
     "rapid_tps": 97.2912753256727,
     "rapid_ttft_s": 0.20252158399671316,
     "task": "Write an email"
    },
    {
     "decode_x": 1.3015283934508475,
     "e2e_x": 1.1490622877292311,
     "first_diff_char": 5,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 8.692362333997153,
     "mlx_tokens": 641,
     "mlx_tps": 75.03538167668695,
     "mlx_ttft_s": 0.14460258299368434,
     "rapid_e2e_s": 7.564744250004878,
     "rapid_tokens": 727,
     "rapid_tps": 97.66067976562952,
     "rapid_ttft_s": 0.15618770898436196,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.4149951821919915,
     "e2e_x": 1.3042373146638153,
     "first_diff_char": 30,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 12.20140445799916,
     "mlx_tokens": 700,
     "mlx_tps": 73.04723399165533,
     "mlx_ttft_s": 2.632221957988804,
     "rapid_e2e_s": 9.355202707985882,
     "rapid_tokens": 700,
     "rapid_tps": 103.36148417064337,
     "rapid_ttft_s": 2.592338499991456,
     "task": "Explain code"
    },
    {
     "decode_x": 1.5149420833960627,
     "e2e_x": 1.4933327966490686,
     "first_diff_char": 1224,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 4.981585666973842,
     "mlx_tokens": 349,
     "mlx_tps": 75.42138380675073,
     "mlx_ttft_s": 0.3544758750067558,
     "rapid_e2e_s": 3.335884458007058,
     "rapid_tokens": 342,
     "rapid_tps": 114.25902831681302,
     "rapid_ttft_s": 0.33520150001277216,
     "task": "SQL"
    },
    {
     "decode_x": 1.443521375751895,
     "e2e_x": 1.4225322885722416,
     "first_diff_char": 80,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 6.167855625011725,
     "mlx_tokens": 450,
     "mlx_tps": 75.24481378624442,
     "mlx_ttft_s": 0.20061483400058933,
     "rapid_e2e_s": 4.335828209004831,
     "rapid_tokens": 450,
     "rapid_tps": 108.6174971149147,
     "rapid_ttft_s": 0.20205424999585375,
     "task": "Regex"
    },
    {
     "decode_x": 1.5633769538760611,
     "e2e_x": 1.4834741160781826,
     "first_diff_char": null,
     "id": "markdown_table",
     "identical": true,
     "mlx_e2e_s": 4.466338542028097,
     "mlx_tokens": 301,
     "mlx_tps": 75.47583979050702,
     "mlx_ttft_s": 0.47885933300131,
     "rapid_e2e_s": 3.0107290000014473,
     "rapid_tokens": 301,
     "rapid_tps": 117.99718850292048,
     "rapid_ttft_s": 0.4702322499942966,
     "task": "Markdown table from notes"
    },
    {
     "decode_x": 1.3490976159712824,
     "e2e_x": 1.4522613827582138,
     "first_diff_char": 87,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 14.862625249981647,
     "mlx_tokens": 1100,
     "mlx_tps": 74.73109280188136,
     "mlx_ttft_s": 0.15661370800808072,
     "rapid_e2e_s": 10.234125500021037,
     "rapid_tokens": 1018,
     "rapid_tps": 100.8195391379468,
     "rapid_ttft_s": 0.19849929100018926,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 20
  },
  "4b_nospec": {
   "best": "code_bugfix_file",
   "config": "rapid_nospec",
   "decode": {
    "geomean": 1.119524481772125,
    "max": 1.1364964951345913,
    "median": 1.119001439845644,
    "min": 1.110234519056083,
    "n": 9
   },
   "e2e": {
    "geomean": 1.1025258371257187,
    "max": 1.1460059944791576,
    "median": 1.1049122808652638,
    "min": 1.0465374009180426,
    "n": 9
   },
   "identical": 1,
   "label": "Qwen3.5-4B 4-bit",
   "mlx_reps": 3,
   "model": "4b",
   "reps": 1,
   "rows": [
    {
     "decode_x": 1.1364964951345913,
     "e2e_x": 1.1391388305931236,
     "first_diff_char": 4090,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 29.511459750006907,
     "mlx_tokens": 1921,
     "mlx_tps": 72.26259515744228,
     "mlx_ttft_s": 2.901122375013074,
     "rapid_e2e_s": 25.906815708003705,
     "rapid_tokens": 1900,
     "rapid_tps": 82.12618612576304,
     "rapid_ttft_s": 2.7693988750106655,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 1.1233834535449594,
     "e2e_x": 1.1460059944791576,
     "first_diff_char": 31,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 4.537891125015449,
     "mlx_tokens": 248,
     "mlx_tps": 74.73671572772957,
     "mlx_ttft_s": 1.2202457500097807,
     "rapid_e2e_s": 3.9597446670231875,
     "rapid_tokens": 230,
     "rapid_tps": 83.95798982082474,
     "rapid_ttft_s": 1.2189350830158219,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.1197711312304794,
     "e2e_x": 1.101256703581413,
     "first_diff_char": null,
     "id": "translate",
     "identical": true,
     "mlx_e2e_s": 2.7637809999869205,
     "mlx_tokens": 184,
     "mlx_tps": 75.76583657328679,
     "mlx_ttft_s": 0.3357772089948412,
     "rapid_e2e_s": 2.509661000018241,
     "rapid_tokens": 184,
     "rapid_tps": 84.84039652829298,
     "rapid_ttft_s": 0.33967920902068727,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 1.119001439845644,
     "e2e_x": 1.0465374009180426,
     "first_diff_char": 513,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 1.7737174160138238,
     "mlx_tokens": 123,
     "mlx_tps": 76.24833406856452,
     "mlx_ttft_s": 0.1611188749957364,
     "rapid_e2e_s": 1.6948437910177745,
     "rapid_tokens": 127,
     "rapid_tps": 85.32199560855537,
     "rapid_ttft_s": 0.20476420799968764,
     "task": "Write an email"
    },
    {
     "decode_x": 1.1230907744066423,
     "e2e_x": 1.0543189139334157,
     "first_diff_char": 5,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 8.692362333997153,
     "mlx_tokens": 641,
     "mlx_tps": 75.03538167668695,
     "mlx_ttft_s": 0.14460258299368434,
     "rapid_e2e_s": 8.244528499984881,
     "rapid_tokens": 681,
     "rapid_tps": 84.27154491516833,
     "rapid_ttft_s": 0.16189287501038052,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.1173649953627223,
     "e2e_x": 1.0979577865009078,
     "first_diff_char": 146,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 12.20140445799916,
     "mlx_tokens": 700,
     "mlx_tps": 73.04723399165533,
     "mlx_ttft_s": 2.632221957988804,
     "rapid_e2e_s": 11.112817458022619,
     "rapid_tokens": 700,
     "rapid_tps": 81.62042227034566,
     "rapid_ttft_s": 2.5485958750068676,
     "task": "Explain code"
    },
    {
     "decode_x": 1.1138881523948745,
     "e2e_x": 1.1291249085685704,
     "first_diff_char": 1224,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 4.981585666973842,
     "mlx_tokens": 349,
     "mlx_tps": 75.42138380675073,
     "mlx_ttft_s": 0.3544758750067558,
     "rapid_e2e_s": 4.411899541999446,
     "rapid_tokens": 342,
     "rapid_tps": 84.01098585956628,
     "rapid_ttft_s": 0.3396871669974644,
     "task": "SQL"
    },
    {
     "decode_x": 1.112705125515459,
     "e2e_x": 1.1078084868188731,
     "first_diff_char": 80,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 6.167855625011725,
     "mlx_tokens": 450,
     "mlx_tps": 75.24481378624442,
     "mlx_ttft_s": 0.20061483400058933,
     "rapid_e2e_s": 5.56761904101586,
     "rapid_tokens": 450,
     "rapid_tps": 83.72528996841044,
     "rapid_ttft_s": 0.2046355000056792,
     "task": "Regex"
    },
    {
     "decode_x": 1.110234519056083,
     "e2e_x": 1.1049122808652638,
     "first_diff_char": 1627,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 14.862625249981647,
     "mlx_tokens": 1100,
     "mlx_tps": 74.73109280188136,
     "mlx_ttft_s": 0.15661370800808072,
     "rapid_e2e_s": 13.451407417014707,
     "rapid_tokens": 1100,
     "rapid_tps": 82.96903887543226,
     "rapid_ttft_s": 0.20527649999712594,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 9
  },
  "9b": {
   "best": "code_rename",
   "best_vs_generate_x": 4.002680293718246,
   "config": "rapid",
   "decode": {
    "geomean": 1.7625985797492205,
    "max": 4.274072932363679,
    "median": 1.5023925960786055,
    "min": 1.2764558166067297,
    "n": 18
   },
   "e2e": {
    "geomean": 1.5252304046771554,
    "max": 3.1257833715070418,
    "median": 1.365628430165478,
    "min": 1.0535043584157413,
    "n": 20
   },
   "generate_tps": 49.576,
   "identical": 11,
   "label": "Qwen3.5-9B 4-bit",
   "mlx_reps": 3,
   "model": "9b",
   "reps": 3,
   "rows": [
    {
     "decode_x": 4.274072932363679,
     "e2e_x": 3.1232464773639133,
     "first_diff_char": null,
     "id": "code_rename",
     "identical": true,
     "mlx_e2e_s": 34.29445033299271,
     "mlx_tokens": 1408,
     "mlx_tps": 46.42805150534358,
     "mlx_ttft_s": 3.947284458001377,
     "rapid_e2e_s": 10.980385500006378,
     "rapid_tokens": 1408,
     "rapid_tps": 198.43687824137578,
     "rapid_ttft_s": 3.8782986250007525,
     "task": "Code edit: rename across a file"
    },
    {
     "decode_x": 4.218816167131833,
     "e2e_x": 3.1257833715070418,
     "first_diff_char": 4109,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 46.31327312497888,
     "mlx_tokens": 1886,
     "mlx_tps": 45.90202512536589,
     "mlx_ttft_s": 5.197077667020494,
     "rapid_e2e_s": 14.816533208009787,
     "rapid_tokens": 1886,
     "rapid_tps": 193.6522057029852,
     "rapid_ttft_s": 5.052555166010279,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 3.853285278935651,
     "e2e_x": 2.965076164892298,
     "first_diff_char": null,
     "id": "apply_change_request",
     "identical": true,
     "mlx_e2e_s": 56.93697204100317,
     "mlx_tokens": 2261,
     "mlx_tps": 45.17713901235394,
     "mlx_ttft_s": 6.776944999990519,
     "rapid_e2e_s": 19.202532708994113,
     "rapid_tokens": 2261,
     "rapid_tps": 174.08040470073294,
     "rapid_ttft_s": 6.220645332999993,
     "task": "Code edit: apply review comments"
    },
    {
     "decode_x": 1.3656380177925544,
     "e2e_x": 1.213106172638448,
     "first_diff_char": null,
     "id": "unified_diff",
     "identical": true,
     "mlx_e2e_s": 11.365366749989334,
     "mlx_tokens": 290,
     "mlx_tps": 46.29677187952387,
     "mlx_ttft_s": 5.098245458008023,
     "rapid_e2e_s": 9.368814540997846,
     "rapid_tokens": 290,
     "rapid_tps": 63.22463177974705,
     "rapid_ttft_s": 4.769368291017599,
     "task": "Code edit: produce a unified diff"
    },
    {
     "decode_x": 1.637104038899342,
     "e2e_x": 1.525976496403755,
     "first_diff_char": null,
     "id": "json_transform",
     "identical": true,
     "mlx_e2e_s": 35.06870029200218,
     "mlx_tokens": 1408,
     "mlx_tps": 45.98833622868536,
     "mlx_ttft_s": 4.424364041973604,
     "rapid_e2e_s": 22.98115362500539,
     "rapid_tokens": 1408,
     "rapid_tps": 75.28769098224174,
     "rapid_ttft_s": 4.243218999996316,
     "task": "JSON transform (rename keys, convert types)"
    },
    {
     "decode_x": 1.5257312334160982,
     "e2e_x": 1.337433993945756,
     "first_diff_char": null,
     "id": "json_to_csv",
     "identical": true,
     "mlx_e2e_s": 15.829885916988133,
     "mlx_tokens": 537,
     "mlx_tps": 46.323041927436215,
     "mlx_ttft_s": 4.2471125409938395,
     "rapid_e2e_s": 11.836012834013673,
     "rapid_tokens": 537,
     "rapid_tps": 70.67651189553288,
     "rapid_ttft_s": 4.146294915990438,
     "task": "Format conversion: JSON to CSV"
    },
    {
     "decode_x": 1.5801946660720425,
     "e2e_x": 1.3468628748971538,
     "first_diff_char": null,
     "id": "json_schema_fill",
     "identical": true,
     "mlx_e2e_s": 8.029273082996951,
     "mlx_tokens": 278,
     "mlx_tps": 46.95172925345829,
     "mlx_ttft_s": 2.108810458012158,
     "rapid_e2e_s": 5.961462917010067,
     "rapid_tokens": 278,
     "rapid_tps": 74.19287212917347,
     "rapid_ttft_s": 2.1833919159835204,
     "task": "Structured output: fill a JSON schema"
    },
    {
     "decode_x": 1.580062461822411,
     "e2e_x": 1.4391671017393202,
     "first_diff_char": null,
     "id": "extract_fields",
     "identical": true,
     "mlx_e2e_s": 9.115332666988252,
     "mlx_tokens": 366,
     "mlx_tps": 47.0470550661387,
     "mlx_ttft_s": 1.3364170420099981,
     "rapid_e2e_s": 6.333755584026221,
     "rapid_tokens": 366,
     "rapid_tps": 74.33728564929764,
     "rapid_ttft_s": 1.3374552080058493,
     "task": "Extract fields from a document"
    },
    {
     "decode_x": null,
     "e2e_x": 1.247277259629741,
     "first_diff_char": null,
     "id": "tool_call",
     "identical": true,
     "mlx_e2e_s": 5.143323957978282,
     "mlx_tokens": 148,
     "mlx_tps": null,
     "mlx_ttft_s": 5.122779875004198,
     "rapid_e2e_s": 4.123641249985667,
     "rapid_tokens": 113,
     "rapid_tps": null,
     "rapid_ttft_s": 4.094833832990844,
     "task": "Tool call (OpenAI tools)"
    },
    {
     "decode_x": null,
     "e2e_x": 1.0535043584157413,
     "first_diff_char": null,
     "id": "agent_step_8k",
     "identical": true,
     "mlx_e2e_s": 28.46141308301594,
     "mlx_tokens": 217,
     "mlx_tps": null,
     "mlx_ttft_s": 28.43928049999522,
     "rapid_e2e_s": 27.01594241699786,
     "rapid_tokens": 213,
     "rapid_tps": null,
     "rapid_ttft_s": 26.98351904199808,
     "task": "Coding-agent turn, ~8k-token context with tool results"
    },
    {
     "decode_x": 1.3601037186350833,
     "e2e_x": 1.1955661418545795,
     "first_diff_char": 230,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 8.382894875016063,
     "mlx_tokens": 292,
     "mlx_tps": 46.997221103490844,
     "mlx_ttft_s": 2.170499750005547,
     "rapid_e2e_s": 7.011652957997285,
     "rapid_tokens": 312,
     "rapid_tps": 63.92109518837311,
     "rapid_ttft_s": 2.1457491659966763,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.4650462680546261,
     "e2e_x": 1.4205995464524763,
     "first_diff_char": 453,
     "id": "translate",
     "identical": false,
     "mlx_e2e_s": 4.5129068339883815,
     "mlx_tokens": 186,
     "mlx_tps": 47.47029108602727,
     "mlx_ttft_s": 0.5949671670095995,
     "rapid_e2e_s": 3.1767621250182856,
     "rapid_tokens": 182,
     "rapid_tps": 69.54617279905104,
     "rapid_ttft_s": 0.5736386670032516,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 2.1836913769130675,
     "e2e_x": 1.8969882104419944,
     "first_diff_char": null,
     "id": "proofread_doc",
     "identical": true,
     "mlx_e2e_s": 10.681987957999809,
     "mlx_tokens": 436,
     "mlx_tps": 47.055196770742164,
     "mlx_ttft_s": 1.415812499995809,
     "rapid_e2e_s": 5.631024957983755,
     "rapid_tokens": 436,
     "rapid_tps": 102.75402742721728,
     "rapid_ttft_s": 1.39438087499002,
     "task": "Proofread a document, return full text"
    },
    {
     "decode_x": 1.3765890394335352,
     "e2e_x": 1.3550118766180985,
     "first_diff_char": 284,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 3.433996042003855,
     "mlx_tokens": 151,
     "mlx_tps": 47.666073741718066,
     "mlx_ttft_s": 0.2665900840074755,
     "rapid_e2e_s": 2.5342922089912463,
     "rapid_tokens": 144,
     "rapid_tps": 65.61659466567973,
     "rapid_ttft_s": 0.3278794170182664,
     "task": "Write an email"
    },
    {
     "decode_x": 1.2764558166067297,
     "e2e_x": 1.175149590530796,
     "first_diff_char": 1051,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 15.91428666599677,
     "mlx_tokens": 737,
     "mlx_tps": 47.06745005410798,
     "mlx_ttft_s": 0.25032270900555886,
     "rapid_e2e_s": 13.542349666997325,
     "rapid_tokens": 800,
     "rapid_tps": 60.07952039441287,
     "rapid_ttft_s": 0.243105082976399,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.400157041571425,
     "e2e_x": 1.2847333690014624,
     "first_diff_char": 1044,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 19.861678917019162,
     "mlx_tokens": 700,
     "mlx_tps": 46.230163573348484,
     "mlx_ttft_s": 4.740426333009964,
     "rapid_e2e_s": 15.459767291991739,
     "rapid_tokens": 700,
     "rapid_tps": 64.72948906022268,
     "rapid_ttft_s": 4.65261874999851,
     "task": "Explain code"
    },
    {
     "decode_x": 1.4866679778278422,
     "e2e_x": 1.4522638675278299,
     "first_diff_char": 600,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 10.163471290987218,
     "mlx_tokens": 450,
     "mlx_tps": 47.00421374436279,
     "mlx_ttft_s": 0.6110996249772143,
     "rapid_e2e_s": 6.998364083992783,
     "rapid_tokens": 450,
     "rapid_tps": 69.87965939671949,
     "rapid_ttft_s": 0.5728018340014387,
     "task": "SQL"
    },
    {
     "decode_x": 1.3913110741282715,
     "e2e_x": 1.3762449837128574,
     "first_diff_char": 11,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 9.890075207978953,
     "mlx_tokens": 450,
     "mlx_tps": 47.05005642674256,
     "mlx_ttft_s": 0.3472657079983037,
     "rapid_e2e_s": 7.186275209009182,
     "rapid_tokens": 450,
     "rapid_tps": 65.46126454488697,
     "rapid_ttft_s": 0.32692233301349916,
     "task": "Regex"
    },
    {
     "decode_x": 1.5181172143293689,
     "e2e_x": 1.4354813138594114,
     "first_diff_char": null,
     "id": "markdown_table",
     "identical": true,
     "mlx_e2e_s": 7.167414124996867,
     "mlx_tokens": 298,
     "mlx_tps": 47.154085241580844,
     "mlx_ttft_s": 0.8485652919916902,
     "rapid_e2e_s": 4.99303895898629,
     "rapid_tokens": 298,
     "rapid_tps": 71.58542853119832,
     "rapid_ttft_s": 0.8162839160067961,
     "task": "Markdown table from notes"
    },
    {
     "decode_x": 1.3554329065256077,
     "e2e_x": 1.3447982564950725,
     "first_diff_char": 48,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 23.73864391699317,
     "mlx_tokens": 1098,
     "mlx_tps": 46.79431499769126,
     "mlx_ttft_s": 0.26288654198287986,
     "rapid_e2e_s": 17.65219712498947,
     "rapid_tokens": 1100,
     "rapid_tps": 63.4265543861955,
     "rapid_ttft_s": 0.32486475000041537,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 20
  },
  "9b_nospec": {
   "best": "code_bugfix_file",
   "config": "rapid_nospec",
   "decode": {
    "geomean": 1.0666743905327156,
    "max": 1.076116646477628,
    "median": 1.0657024446253973,
    "min": 1.0639050389961364,
    "n": 9
   },
   "e2e": {
    "geomean": 1.0454559854974266,
    "max": 1.0792161066744388,
    "median": 1.0615392794614615,
    "min": 0.9830939892633783,
    "n": 9
   },
   "identical": 0,
   "label": "Qwen3.5-9B 4-bit",
   "mlx_reps": 3,
   "model": "9b",
   "reps": 1,
   "rows": [
    {
     "decode_x": 1.076116646477628,
     "e2e_x": 1.063760448852175,
     "first_diff_char": 4109,
     "id": "code_bugfix_file",
     "identical": false,
     "mlx_e2e_s": 46.31327312497888,
     "mlx_tokens": 1886,
     "mlx_tps": 45.90202512536589,
     "mlx_ttft_s": 5.197077667020494,
     "rapid_e2e_s": 43.53731441599666,
     "rapid_tokens": 1899,
     "rapid_tps": 49.395933344440564,
     "rapid_ttft_s": 5.091016624995973,
     "task": "Code edit: fix a bug, return whole file"
    },
    {
     "decode_x": 1.0677242527984299,
     "e2e_x": 0.9961558208230078,
     "first_diff_char": 239,
     "id": "summarize",
     "identical": false,
     "mlx_e2e_s": 8.382894875016063,
     "mlx_tokens": 292,
     "mlx_tps": 46.997221103490844,
     "mlx_ttft_s": 2.170499750005547,
     "rapid_e2e_s": 8.415244583011372,
     "rapid_tokens": 314,
     "rapid_tps": 50.18007278632736,
     "rapid_ttft_s": 2.1556998750020284,
     "task": "Summarize an article"
    },
    {
     "decode_x": 1.0656561842322043,
     "e2e_x": 1.058622323788535,
     "first_diff_char": 453,
     "id": "translate",
     "identical": false,
     "mlx_e2e_s": 4.5129068339883815,
     "mlx_tokens": 186,
     "mlx_tps": 47.47029108602727,
     "mlx_ttft_s": 0.5949671670095995,
     "rapid_e2e_s": 4.262999875005335,
     "rapid_tokens": 186,
     "rapid_tps": 50.58700926312784,
     "rapid_ttft_s": 0.5843252500053495,
     "task": "Translate (English to Spanish)"
    },
    {
     "decode_x": 1.0639505155779956,
     "e2e_x": 1.0792161066744388,
     "first_diff_char": 284,
     "id": "write_email",
     "identical": false,
     "mlx_e2e_s": 3.433996042003855,
     "mlx_tokens": 151,
     "mlx_tps": 47.666073741718066,
     "mlx_ttft_s": 0.2665900840074755,
     "rapid_e2e_s": 3.181935500004329,
     "rapid_tokens": 144,
     "rapid_tps": 50.714343733079694,
     "rapid_ttft_s": 0.34053524999762885,
     "task": "Write an email"
    },
    {
     "decode_x": 1.0657024446253973,
     "e2e_x": 0.9830939892633783,
     "first_diff_char": 50,
     "id": "story",
     "identical": false,
     "mlx_e2e_s": 15.91428666599677,
     "mlx_tokens": 737,
     "mlx_tps": 47.06745005410798,
     "mlx_ttft_s": 0.25032270900555886,
     "rapid_e2e_s": 16.187960500014015,
     "rapid_tokens": 799,
     "rapid_tps": 50.159896584946665,
     "rapid_ttft_s": 0.2561668330163229,
     "task": "Free-form prose (short story)"
    },
    {
     "decode_x": 1.0667542658247866,
     "e2e_x": 1.046418205307982,
     "first_diff_char": 1044,
     "id": "explain_code",
     "identical": false,
     "mlx_e2e_s": 19.861678917019162,
     "mlx_tokens": 700,
     "mlx_tps": 46.230163573348484,
     "mlx_ttft_s": 4.740426333009964,
     "rapid_e2e_s": 18.980632042017533,
     "rapid_tokens": 700,
     "rapid_tps": 49.31622420164715,
     "rapid_ttft_s": 4.806613458000356,
     "task": "Explain code"
    },
    {
     "decode_x": 1.0639255574196238,
     "e2e_x": 1.0619154150544847,
     "first_diff_char": 1610,
     "id": "sql",
     "identical": false,
     "mlx_e2e_s": 10.163471290987218,
     "mlx_tokens": 450,
     "mlx_tps": 47.00421374436279,
     "mlx_ttft_s": 0.6110996249772143,
     "rapid_e2e_s": 9.570885916997213,
     "rapid_tokens": 450,
     "rapid_tps": 50.00898430904232,
     "rapid_ttft_s": 0.5922854170203209,
     "task": "SQL"
    },
    {
     "decode_x": 1.0639050389961364,
     "e2e_x": 1.062685798713797,
     "first_diff_char": 732,
     "id": "regex",
     "identical": false,
     "mlx_e2e_s": 9.890075207978953,
     "mlx_tokens": 450,
     "mlx_tps": 47.05005642674256,
     "mlx_ttft_s": 0.3472657079983037,
     "rapid_e2e_s": 9.306678624998312,
     "rapid_tokens": 450,
     "rapid_tps": 50.056792117463964,
     "rapid_ttft_s": 0.3366357079939917,
     "task": "Regex"
    },
    {
     "decode_x": 1.0663882755343708,
     "e2e_x": 1.0615392794614615,
     "first_diff_char": 48,
     "id": "long_answer",
     "identical": false,
     "mlx_e2e_s": 23.73864391699317,
     "mlx_tokens": 1098,
     "mlx_tps": 46.79431499769126,
     "mlx_ttft_s": 0.26288654198287986,
     "rapid_e2e_s": 22.362473415996647,
     "rapid_tokens": 1100,
     "rapid_tps": 49.90090887520012,
     "rapid_ttft_s": 0.33861816601711325,
     "task": "Long technical answer (~800 tokens)"
    }
   ],
   "tasks": 9
  }
 },
 "source": "computed by landing/scripts/vs_mlx.py from raw/ (see README.md)",
 "task_count": 20
}