{ "schema_version": 1, "generated_at": "2026-06-18T01:31:05.308941+00:00", "source_rows": 10, "groups": [ { "chip": "Apple M3 Ultra", "model_alias": "gemma3-1b-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "bcdd015e6401" ], "short": { "decode_tps": { "median": 257.65444178143696, "p25": 257.65444178143696, "p75": 257.65444178143696 }, "prefill_tps": { "median": 6827.53063250403, "p25": 6827.53063250403, "p75": 6827.53063250403 }, "ttft_ms": { "median": 78.06629200058524, "p25": 78.06629200058524, "p75": 78.06629200058524 } }, "long": { "decode_tps": { "median": 258.6301577366272, "p25": 258.6301577366272, "p75": 258.6301577366272 }, "prefill_tps": { "median": 8345.765070368108, "p25": 8345.765070368108, "p75": 8345.765070368108 }, "ttft_ms": { "median": 255.33908299985342, "p25": 255.33908299985342, "p75": 255.33908299985342 } } }, { "chip": "Apple M3 Ultra", "model_alias": "gemma3-1b-qat-4bit", "rapid_mlx_version": "0.7.25", "row_count": 1, "submission_ids": [ "2a31e18c1a47" ], "short": { "decode_tps": { "median": 255.62226426810216, "p25": 255.62226426810216, "p75": 255.62226426810216 }, "prefill_tps": { "median": 7321.374063889859, "p25": 7321.374063889859, "p75": 7321.374063889859 }, "ttft_ms": { "median": 72.80054199509323, "p25": 72.80054199509323, "p75": 72.80054199509323 } }, "long": { "decode_tps": { "median": 257.79767007228963, "p25": 257.79767007228963, "p75": 257.79767007228963 }, "prefill_tps": { "median": 8722.691289235543, "p25": 8722.691289235543, "p75": 8722.691289235543 }, "ttft_ms": { "median": 244.3053329916438, "p25": 244.3053329916438, "p75": 244.3053329916438 } } }, { "chip": "Apple M3 Ultra", "model_alias": "gemma3-1b-qat-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "ba03a40c2ad9" ], "short": { "decode_tps": { "median": 261.0653644230458, "p25": 261.0653644230458, "p75": 261.0653644230458 }, "prefill_tps": { "median": 7356.125130118277, "p25": 7356.125130118277, "p75": 7356.125130118277 }, "ttft_ms": { "median": 72.45662499917671, "p25": 72.45662499917671, "p75": 72.45662499917671 } }, "long": { "decode_tps": { "median": 262.0431016038515, "p25": 262.0431016038515, "p75": 262.0431016038515 }, "prefill_tps": { "median": 8737.021694177374, "p25": 8737.021694177374, "p75": 8737.021694177374 }, "ttft_ms": { "median": 243.90462500741705, "p25": 243.90462500741705, "p75": 243.90462500741705 } } }, { "chip": "Apple M3 Ultra", "model_alias": "gemma3-27b-qat-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "e3521ff5f627" ], "short": { "decode_tps": { "median": 35.1881120367345, "p25": 35.1881120367345, "p75": 35.1881120367345 }, "prefill_tps": { "median": 298.6041794686046, "p25": 298.6041794686046, "p75": 298.6041794686046 }, "ttft_ms": { "median": 1784.9716670025373, "p25": 1784.9716670025373, "p75": 1784.9716670025373 } }, "long": { "decode_tps": { "median": 34.939599562450496, "p25": 34.939599562450496, "p75": 34.939599562450496 }, "prefill_tps": { "median": 310.25002534141936, "p25": 310.25002534141936, "p75": 310.25002534141936 }, "ttft_ms": { "median": 6868.653750003432, "p25": 6868.653750003432, "p75": 6868.653750003432 } } }, { "chip": "Apple M3 Ultra", "model_alias": "gemma3-4b-qat-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "150bbadafcd4" ], "short": { "decode_tps": { "median": 141.15799375940261, "p25": 141.15799375940261, "p75": 141.15799375940261 }, "prefill_tps": { "median": 2034.0467522793213, "p25": 2034.0467522793213, "p75": 2034.0467522793213 }, "ttft_ms": { "median": 262.039207998896, "p25": 262.039207998896, "p75": 262.039207998896 } }, "long": { "decode_tps": { "median": 152.53395540919678, "p25": 152.53395540919678, "p75": 152.53395540919678 }, "prefill_tps": { "median": 2206.224249353056, "p25": 2206.224249353056, "p75": 2206.224249353056 }, "ttft_ms": { "median": 965.9036249940982, "p25": 965.9036249940982, "p75": 965.9036249940982 } } }, { "chip": "Apple M3 Ultra", "model_alias": "llama-3.1-8b-4bit", "rapid_mlx_version": "0.7.9", "row_count": 1, "submission_ids": [ "acd2ae43768f" ], "short": { "decode_tps": { "median": 119.39102110938477, "p25": 119.39102110938477, "p75": 119.39102110938477 }, "prefill_tps": { "median": 1155.9836246559605, "p25": 1155.9836246559605, "p75": 1155.9836246559605 }, "ttft_ms": { "median": 456.7538750015956, "p25": 456.7538750015956, "p75": 456.7538750015956 } }, "long": { "decode_tps": { "median": 110.44850712553149, "p25": 110.44850712553149, "p75": 110.44850712553149 }, "prefill_tps": { "median": 1193.6474648339151, "p25": 1193.6474648339151, "p75": 1193.6474648339151 }, "ttft_ms": { "median": 1793.6619170031918, "p25": 1793.6619170031918, "p75": 1793.6619170031918 } } }, { "chip": "Apple M3 Ultra", "model_alias": "phi-4-mini-4bit", "rapid_mlx_version": "0.7.9", "row_count": 1, "submission_ids": [ "cc3b1481796d" ], "short": { "decode_tps": { "median": 167.38130466134575, "p25": 167.38130466134575, "p75": 167.38130466134575 }, "prefill_tps": { "median": 2245.6056782481996, "p25": 2245.6056782481996, "p75": 2245.6056782481996 }, "ttft_ms": { "median": 235.12587499863002, "p25": 235.12587499863002, "p75": 235.12587499863002 } }, "long": { "decode_tps": { "median": 159.38537958390728, "p25": 159.38537958390728, "p75": 159.38537958390728 }, "prefill_tps": { "median": 2327.972978355838, "p25": 2327.972978355838, "p75": 2327.972978355838 }, "ttft_ms": { "median": 911.9521659995371, "p25": 911.9521659995371, "p75": 911.9521659995371 } } }, { "chip": "Apple M3 Ultra", "model_alias": "qwen2.5-14b-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "94b9aedd6522" ], "short": { "decode_tps": { "median": 66.36243007407856, "p25": 66.36243007407856, "p75": 66.36243007407856 }, "prefill_tps": { "median": 606.9246849037439, "p25": 606.9246849037439, "p75": 606.9246849037439 }, "ttft_ms": { "median": 874.9026250006864, "p25": 874.9026250006864, "p75": 874.9026250006864 } }, "long": { "decode_tps": { "median": 60.467997455139184, "p25": 60.467997455139184, "p75": 60.467997455139184 }, "prefill_tps": { "median": 622.2664907049742, "p25": 622.2664907049742, "p75": 622.2664907049742 }, "ttft_ms": { "median": 3469.574582995847, "p25": 3469.574582995847, "p75": 3469.574582995847 } } }, { "chip": "Apple M3 Ultra", "model_alias": "qwen3-0.6b-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "eb0f689d7b51" ], "short": { "decode_tps": { "median": 387.64067235263275, "p25": 387.64067235263275, "p75": 387.64067235263275 }, "prefill_tps": { "median": 8853.725941652827, "p25": 8853.725941652827, "p75": 8853.725941652827 }, "ttft_ms": { "median": 59.97475000913255, "p25": 59.97475000913255, "p75": 59.97475000913255 } }, "long": { "decode_tps": { "median": 340.1172734206654, "p25": 340.1172734206654, "p75": 340.1172734206654 }, "prefill_tps": { "median": 9353.220513064718, "p25": 9353.220513064718, "p75": 9353.220513064718 }, "ttft_ms": { "median": 230.8295839902712, "p25": 230.8295839902712, "p75": 230.8295839902712 } } }, { "chip": "Apple M3 Ultra", "model_alias": "qwen3-8b-4bit", "rapid_mlx_version": "0.7.26", "row_count": 1, "submission_ids": [ "e912922f4600" ], "short": { "decode_tps": { "median": 112.8930742617306, "p25": 112.8930742617306, "p75": 112.8930742617306 }, "prefill_tps": { "median": 1069.1567578596164, "p25": 1069.1567578596164, "p75": 1069.1567578596164 }, "ttft_ms": { "median": 496.6530829988187, "p25": 496.6530829988187, "p75": 496.6530829988187 } }, "long": { "decode_tps": { "median": 103.74271555462862, "p25": 103.74271555462862, "p75": 103.74271555462862 }, "prefill_tps": { "median": 1093.5023434295824, "p25": 1093.5023434295824, "p75": 1093.5023434295824 }, "ttft_ms": { "median": 1974.389916009386, "p25": 1974.389916009386, "p75": 1974.389916009386 } } } ] }