{ "models": [ { "aic": { "points": 84, "tpot_mape_pct": 18.38, "tpot_shape_error_pct": 13.02, "ttft_mape_pct": 47.16, "ttft_shape_error_pct": 40.7 }, "aisimulate": { "coverage_pct": 95.24, "points": 80, "status_counts": { "failed": 4, "success": 80, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 19.24, "tpot_shape_error_pct": 13.15, "ttft_mape_pct": 57.63, "ttft_shape_error_pct": 30.67 }, "frameworks": [ "sglang", "trtllm" ], "gpu_skus": [ "b200", "b300", "h200" ], "hf_model_paths": [ "deepseek-ai/DeepSeek-V3" ], "model": "DeepSeek-R1-0528", "precisions": [ "fp4", "fp8" ], "rows": 84, "workloads": [ { "aic": { "points": 33, "tpot_mape_pct": 14.28, "tpot_shape_error_pct": 12.76, "ttft_mape_pct": 46.29, "ttft_shape_error_pct": 34.55 }, "aisimulate": { "coverage_pct": 100.0, "points": 33, "status_counts": { "failed": 0, "success": 33, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.3, "tpot_shape_error_pct": 13.05, "ttft_mape_pct": 67.25, "ttft_shape_error_pct": 28.76 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 21, "tpot_mape_pct": 14.01, "tpot_shape_error_pct": 15.7, "ttft_mape_pct": 53.93, "ttft_shape_error_pct": 33.78 }, "aisimulate": { "coverage_pct": 100.0, "points": 21, "status_counts": { "failed": 0, "success": 21, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.39, "tpot_shape_error_pct": 16.24, "ttft_mape_pct": 69.61, "ttft_shape_error_pct": 24.45 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 21, "topologies": [ { "aic": { "points": 2, "tpot_mape_pct": 22.36, "tpot_shape_error_pct": 1.77, "ttft_mape_pct": 97.85, "ttft_shape_error_pct": 0.78 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.46, "tpot_shape_error_pct": 1.15, "ttft_mape_pct": 98.45, "ttft_shape_error_pct": 53.65 }, "framework": "sglang", "id": "ce15f83b8bd12f26", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 21.67, "tpot_relative": 0.783297, "ttft_error_pct": 97.86, "ttft_relative": 0.02142 }, "aisimulate": { "tpot_error_pct": 18.93, "tpot_relative": 0.810741, "ttft_error_pct": 98.78, "ttft_relative": 0.012221 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.06, "tpot_relative": 0.978775, "ttft_error_pct": 97.84, "ttft_relative": 0.021884 }, "aisimulate": { "tpot_error_pct": 18.0, "tpot_relative": 1.043184, "ttft_error_pct": 98.12, "ttft_relative": 0.019037 }, "concurrency": 128, "measured": { "tpot_relative": 1.272114, "ttft_relative": 1.013782 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 8.22, "tpot_shape_error_pct": 11.33, "ttft_mape_pct": 60.04, "ttft_shape_error_pct": 30.93 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.11, "tpot_shape_error_pct": 9.89, "ttft_mape_pct": 79.72, "ttft_shape_error_pct": 29.29 }, "framework": "sglang", "id": "f3d10bc03f1cdbb2", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 2.41, "tpot_relative": 1.02409, "ttft_error_pct": 45.62, "ttft_relative": 0.54378 }, "aisimulate": { "tpot_error_pct": 2.16, "tpot_relative": 1.021616, "ttft_error_pct": 72.93, "ttft_relative": 0.270742 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.41, "tpot_relative": 0.988772, "ttft_error_pct": 48.64, "ttft_relative": 0.569758 }, "aisimulate": { "tpot_error_pct": 10.29, "tpot_relative": 0.990099, "ttft_error_pct": 74.67, "ttft_relative": 0.280975 }, "concurrency": 2, "measured": { "tpot_relative": 1.103622, "ttft_relative": 1.109423 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.67, "tpot_relative": 1.121083, "ttft_error_pct": 52.87, "ttft_relative": 0.599177 }, "aisimulate": { "tpot_error_pct": 7.9, "tpot_relative": 1.130487, "ttft_error_pct": 76.25, "ttft_relative": 0.301895 }, "concurrency": 4, "measured": { "tpot_relative": 1.227522, "ttft_relative": 1.271354 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.99, "tpot_relative": 1.418878, "ttft_error_pct": 57.58, "ttft_relative": 0.658072 }, "aisimulate": { "tpot_error_pct": 6.42, "tpot_relative": 1.427467, "ttft_error_pct": 77.8, "ttft_relative": 0.344417 }, "concurrency": 8, "measured": { "tpot_relative": 1.52547, "ttft_relative": 1.551167 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.94, "tpot_relative": 1.677154, "ttft_error_pct": 66.23, "ttft_relative": 0.776097 }, "aisimulate": { "tpot_error_pct": 6.81, "tpot_relative": 1.697674, "ttft_error_pct": 82.99, "ttft_relative": 0.390794 }, "concurrency": 16, "measured": { "tpot_relative": 1.821799, "ttft_relative": 2.297922 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.08, "tpot_relative": 2.029363, "ttft_error_pct": 65.5, "ttft_relative": 1.013085 }, "aisimulate": { "tpot_error_pct": 8.28, "tpot_relative": 2.069969, "ttft_error_pct": 83.07, "ttft_relative": 0.497334 }, "concurrency": 32, "measured": { "tpot_relative": 2.256806, "ttft_relative": 2.936897 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.06, "tpot_relative": 2.690424, "ttft_error_pct": 83.82, "ttft_relative": 1.180841 }, "aisimulate": { "tpot_error_pct": 7.93, "tpot_relative": 2.785329, "ttft_error_pct": 90.35, "ttft_relative": 0.704152 }, "concurrency": 64, "measured": { "tpot_relative": 3.025092, "ttft_relative": 7.2981 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 14.57, "tpot_shape_error_pct": 16.69, "ttft_mape_pct": 34.05, "ttft_shape_error_pct": 36.87 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.4, "tpot_shape_error_pct": 19.02, "ttft_mape_pct": 52.99, "ttft_shape_error_pct": 16.59 }, "framework": "trtllm", "id": "a1b78c73ed533ad5", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 10.72, "tpot_relative": 1.107234, "ttft_error_pct": 18.69, "ttft_relative": 0.813087 }, "aisimulate": { "tpot_error_pct": 11.78, "tpot_relative": 1.117789, "ttft_error_pct": 52.75, "ttft_relative": 0.472502 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.3, "tpot_relative": 1.292877, "ttft_error_pct": 6.51, "ttft_relative": 0.893751 }, "aisimulate": { "tpot_error_pct": 6.39, "tpot_relative": 1.306316, "ttft_error_pct": 43.95, "ttft_relative": 0.535881 }, "concurrency": 8, "measured": { "tpot_relative": 1.227858, "ttft_relative": 0.956005 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.93, "tpot_relative": 1.582638, "ttft_error_pct": 2.01, "ttft_relative": 1.055795 }, "aisimulate": { "tpot_error_pct": 4.44, "tpot_relative": 1.605771, "ttft_error_pct": 42.69, "ttft_relative": 0.617451 }, "concurrency": 16, "measured": { "tpot_relative": 1.537555, "ttft_relative": 1.077448 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.4, "tpot_relative": 2.724239, "ttft_error_pct": 42.98, "ttft_relative": 1.622269 }, "aisimulate": { "tpot_error_pct": 4.57, "tpot_relative": 2.807493, "ttft_error_pct": 61.96, "ttft_relative": 1.082178 }, "concurrency": 64, "measured": { "tpot_relative": 2.941833, "ttft_relative": 2.845152 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.63, "tpot_relative": 3.529277, "ttft_error_pct": 57.54, "ttft_relative": 1.660558 }, "aisimulate": { "tpot_error_pct": 2.33, "tpot_relative": 3.691681, "ttft_error_pct": 57.38, "ttft_relative": 1.666701 }, "concurrency": 128, "measured": { "tpot_relative": 3.779711, "ttft_relative": 3.910775 }, "status": "success" }, { "aic": { "tpot_error_pct": 54.45, "tpot_relative": 7.068914, "ttft_error_pct": 76.57, "ttft_relative": 1.737391 }, "aisimulate": { "tpot_error_pct": 74.91, "tpot_relative": 8.005308, "ttft_error_pct": 59.23, "ttft_relative": 3.023602 }, "concurrency": 256, "measured": { "tpot_relative": 4.576825, "ttft_relative": 7.416587 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 17.42, "tpot_shape_error_pct": 22.75, "ttft_mape_pct": 52.05, "ttft_shape_error_pct": 40.73 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.52, "tpot_shape_error_pct": 24.12, "ttft_mape_pct": 64.8, "ttft_shape_error_pct": 20.67 }, "framework": "trtllm", "id": "6a537fc093722375", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 10.7, "tpot_relative": 1.106959, "ttft_error_pct": 39.93, "ttft_relative": 0.600733 }, "aisimulate": { "tpot_error_pct": 11.9, "tpot_relative": 1.119022, "ttft_error_pct": 63.73, "ttft_relative": 0.362658 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 19.83, "tpot_relative": 1.483571, "ttft_error_pct": 23.94, "ttft_relative": 0.780674 }, "aisimulate": { "tpot_error_pct": 18.74, "tpot_relative": 1.503746, "ttft_error_pct": 53.21, "ttft_relative": 0.480194 }, "concurrency": 16, "measured": { "tpot_relative": 1.850499, "ttft_relative": 1.026338 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.7, "tpot_relative": 1.850215, "ttft_error_pct": 31.14, "ttft_relative": 1.023498 }, "aisimulate": { "tpot_error_pct": 13.07, "tpot_relative": 1.885581, "ttft_error_pct": 58.72, "ttft_relative": 0.613611 }, "concurrency": 32, "measured": { "tpot_relative": 2.169059, "ttft_relative": 1.486317 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.55, "tpot_relative": 2.473471, "ttft_error_pct": 61.0, "ttft_relative": 1.203251 }, "aisimulate": { "tpot_error_pct": 13.04, "tpot_relative": 2.546823, "ttft_error_pct": 71.97, "ttft_relative": 0.864657 }, "concurrency": 64, "measured": { "tpot_relative": 2.928883, "ttft_relative": 3.085228 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.3, "tpot_relative": 3.256869, "ttft_error_pct": 70.79, "ttft_relative": 1.23646 }, "aisimulate": { "tpot_error_pct": 10.81, "tpot_relative": 3.429409, "ttft_error_pct": 68.0, "ttft_relative": 1.354854 }, "concurrency": 128, "measured": { "tpot_relative": 3.845131, "ttft_relative": 4.233459 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.45, "tpot_relative": 5.487206, "ttft_error_pct": 85.53, "ttft_relative": 1.303908 }, "aisimulate": { "tpot_error_pct": 43.56, "tpot_relative": 6.132535, "ttft_error_pct": 73.18, "ttft_relative": 2.417405 }, "concurrency": 256, "measured": { "tpot_relative": 4.271819, "ttft_relative": 9.013294 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 7, "tpot_mape_pct": 6.91, "tpot_shape_error_pct": 1.67, "ttft_mape_pct": 41.64, "ttft_shape_error_pct": 54.16 }, "aisimulate": { "coverage_pct": 100.0, "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.91, "tpot_shape_error_pct": 1.62, "ttft_mape_pct": 67.95, "ttft_shape_error_pct": 53.54 }, "gpu": "b300", "precisions": [ "fp8" ], "rows": 7, "topologies": [ { "aic": { "points": 7, "tpot_mape_pct": 6.91, "tpot_shape_error_pct": 1.67, "ttft_mape_pct": 41.64, "ttft_shape_error_pct": 54.16 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.91, "tpot_shape_error_pct": 1.62, "ttft_mape_pct": 67.95, "ttft_shape_error_pct": 53.54 }, "framework": "sglang", "id": "9619f4e979a2c435", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 6.17, "tpot_relative": 0.9383, "ttft_error_pct": 19.12, "ttft_relative": 1.19124 }, "aisimulate": { "tpot_error_pct": 6.2, "tpot_relative": 0.937961, "ttft_error_pct": 40.77, "ttft_relative": 0.592348 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.42, "tpot_relative": 0.982766, "ttft_error_pct": 23.41, "ttft_relative": 1.266345 }, "aisimulate": { "tpot_error_pct": 9.48, "tpot_relative": 0.982132, "ttft_error_pct": 62.81, "ttft_relative": 0.614948 }, "concurrency": 2, "measured": { "tpot_relative": 1.084935, "ttft_relative": 1.653369 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.09, "tpot_relative": 1.118451, "ttft_error_pct": 26.75, "ttft_relative": 1.331624 }, "aisimulate": { "tpot_error_pct": 4.34, "tpot_relative": 1.127287, "ttft_error_pct": 63.78, "ttft_relative": 0.658511 }, "concurrency": 4, "measured": { "tpot_relative": 1.178423, "ttft_relative": 1.81794 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.04, "tpot_relative": 1.349892, "ttft_error_pct": 29.0, "ttft_relative": 1.462281 }, "aisimulate": { "tpot_error_pct": 5.5, "tpot_relative": 1.357744, "ttft_error_pct": 63.45, "ttft_relative": 0.752718 }, "concurrency": 8, "measured": { "tpot_relative": 1.436708, "ttft_relative": 2.05966 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.89, "tpot_relative": 1.633326, "ttft_error_pct": 60.85, "ttft_relative": 1.723989 }, "aisimulate": { "tpot_error_pct": 5.8, "tpot_relative": 1.652364, "ttft_error_pct": 80.61, "ttft_relative": 0.853647 }, "concurrency": 16, "measured": { "tpot_relative": 1.754119, "ttft_relative": 4.40344 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.28, "tpot_relative": 2.079803, "ttft_error_pct": 62.12, "ttft_relative": 2.248981 }, "aisimulate": { "tpot_error_pct": 3.62, "tpot_relative": 2.116241, "ttft_error_pct": 81.7, "ttft_relative": 1.086616 }, "concurrency": 32, "measured": { "tpot_relative": 2.195676, "ttft_relative": 5.936451 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.52, "tpot_relative": 2.719524, "ttft_error_pct": 70.23, "ttft_relative": 2.618015 }, "aisimulate": { "tpot_error_pct": 6.46, "tpot_relative": 2.811464, "ttft_error_pct": 82.51, "ttft_relative": 1.537588 }, "concurrency": 64, "measured": { "tpot_relative": 3.005708, "ttft_relative": 8.793193 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 5, "tpot_mape_pct": 25.7, "tpot_shape_error_pct": 16.91, "ttft_mape_pct": 20.71, "ttft_shape_error_pct": 8.37 }, "aisimulate": { "coverage_pct": 100.0, "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 25.66, "tpot_shape_error_pct": 16.65, "ttft_mape_pct": 56.38, "ttft_shape_error_pct": 9.9 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 5, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 25.7, "tpot_shape_error_pct": 16.91, "ttft_mape_pct": 20.71, "ttft_shape_error_pct": 8.37 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 25.66, "tpot_shape_error_pct": 16.65, "ttft_mape_pct": 56.38, "ttft_shape_error_pct": 9.9 }, "framework": "sglang", "id": "1ca6d8ac2af36927", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 10.72, "tpot_relative": 1.107237, "ttft_error_pct": 17.99, "ttft_relative": 0.820141 }, "aisimulate": { "tpot_error_pct": 10.88, "tpot_relative": 1.108829, "ttft_error_pct": 58.01, "ttft_relative": 0.419852 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.89, "tpot_relative": 1.389316, "ttft_error_pct": 13.99, "ttft_relative": 0.901542 }, "aisimulate": { "tpot_error_pct": 14.6, "tpot_relative": 1.385778, "ttft_error_pct": 53.31, "ttft_relative": 0.489361 }, "concurrency": 8, "measured": { "tpot_relative": 1.209255, "ttft_relative": 1.048123 }, "status": "success" }, { "aic": { "tpot_error_pct": 36.28, "tpot_relative": 2.147592, "ttft_error_pct": 15.07, "ttft_relative": 1.065086 }, "aisimulate": { "tpot_error_pct": 35.09, "tpot_relative": 2.12879, "ttft_error_pct": 55.08, "ttft_relative": 0.563334 }, "concurrency": 16, "measured": { "tpot_relative": 1.575818, "ttft_relative": 1.254011 }, "status": "success" }, { "aic": { "tpot_error_pct": 31.48, "tpot_relative": 2.954934, "ttft_error_pct": 28.31, "ttft_relative": 1.395147 }, "aisimulate": { "tpot_error_pct": 31.09, "tpot_relative": 2.946016, "ttft_error_pct": 62.24, "ttft_relative": 0.734873 }, "concurrency": 32, "measured": { "tpot_relative": 2.247366, "ttft_relative": 1.946008 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.13, "tpot_relative": 4.146924, "ttft_error_pct": 28.19, "ttft_relative": 1.637387 }, "aisimulate": { "tpot_error_pct": 36.62, "tpot_relative": 4.192785, "ttft_error_pct": 53.24, "ttft_relative": 1.066155 }, "concurrency": 64, "measured": { "tpot_relative": 3.068897, "ttft_relative": 2.280286 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4", "fp8" ], "rows": 33 }, { "aic": { "points": 12, "tpot_mape_pct": 17.98, "tpot_shape_error_pct": 10.94, "ttft_mape_pct": 72.68, "ttft_shape_error_pct": 56.18 }, "aisimulate": { "coverage_pct": 100.0, "points": 12, "status_counts": { "failed": 0, "success": 12, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.81, "tpot_shape_error_pct": 11.39, "ttft_mape_pct": 79.44, "ttft_shape_error_pct": 43.05 }, "gpu_skus": [ "b200" ], "gpus": [ { "aic": { "points": 12, "tpot_mape_pct": 17.98, "tpot_shape_error_pct": 10.94, "ttft_mape_pct": 72.68, "ttft_shape_error_pct": 56.18 }, "aisimulate": { "coverage_pct": 100.0, "points": 12, "status_counts": { "failed": 0, "success": 12, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.81, "tpot_shape_error_pct": 11.39, "ttft_mape_pct": 79.44, "ttft_shape_error_pct": 43.05 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 12, "topologies": [ { "aic": { "points": 6, "tpot_mape_pct": 20.64, "tpot_shape_error_pct": 5.76, "ttft_mape_pct": 91.67, "ttft_shape_error_pct": 57.97 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 21.45, "tpot_shape_error_pct": 6.2, "ttft_mape_pct": 95.67, "ttft_shape_error_pct": 55.81 }, "framework": "sglang", "id": "fb6bbd3d16911787", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 16.64, "tpot_relative": 0.833629, "ttft_error_pct": 86.73, "ttft_relative": 0.132688 }, "aisimulate": { "tpot_error_pct": 17.17, "tpot_relative": 0.828292, "ttft_error_pct": 93.25, "ttft_relative": 0.067541 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.49, "tpot_relative": 0.908377, "ttft_error_pct": 86.71, "ttft_relative": 0.145833 }, "aisimulate": { "tpot_error_pct": 24.93, "tpot_relative": 0.903052, "ttft_error_pct": 93.11, "ttft_relative": 0.075554 }, "concurrency": 8, "measured": { "tpot_relative": 1.202959, "ttft_relative": 1.097056 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.52, "tpot_relative": 1.121306, "ttft_error_pct": 82.35, "ttft_relative": 0.172231 }, "aisimulate": { "tpot_error_pct": 23.11, "tpot_relative": 1.112714, "ttft_error_pct": 91.21, "ttft_relative": 0.085758 }, "concurrency": 16, "measured": { "tpot_relative": 1.447218, "ttft_relative": 0.976081 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.68, "tpot_relative": 1.442268, "ttft_error_pct": 98.25, "ttft_relative": 0.225456 }, "aisimulate": { "tpot_error_pct": 23.68, "tpot_relative": 1.423633, "ttft_error_pct": 99.18, "ttft_relative": 0.10626 }, "concurrency": 32, "measured": { "tpot_relative": 1.865356, "ttft_relative": 12.918044 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.17, "tpot_relative": 1.95654, "ttft_error_pct": 97.79, "ttft_relative": 0.264255 }, "aisimulate": { "tpot_error_pct": 18.36, "tpot_relative": 1.9284, "ttft_error_pct": 98.77, "ttft_relative": 0.147598 }, "concurrency": 64, "measured": { "tpot_relative": 2.362023, "ttft_relative": 11.953453 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.35, "tpot_relative": 2.502643, "ttft_error_pct": 98.21, "ttft_relative": 0.269984 }, "aisimulate": { "tpot_error_pct": 21.46, "tpot_relative": 2.467689, "ttft_error_pct": 98.48, "ttft_relative": 0.229223 }, "concurrency": 128, "measured": { "tpot_relative": 3.141992, "ttft_relative": 15.062961 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 12.35, "tpot_shape_error_pct": 5.82, "ttft_mape_pct": 44.19, "ttft_shape_error_pct": 49.48 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.52, "tpot_shape_error_pct": 5.98, "ttft_mape_pct": 56.7, "ttft_shape_error_pct": 23.99 }, "framework": "trtllm", "id": "b818b9f29780b771", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 9.72, "tpot_relative": 0.902758, "ttft_error_pct": 25.85, "ttft_relative": 0.741549 }, "aisimulate": { "tpot_error_pct": 10.85, "tpot_relative": 0.891458, "ttft_error_pct": 50.8, "ttft_relative": 0.491968 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.97, "tpot_relative": 1.151812, "ttft_error_pct": 62.54, "ttft_relative": 0.759051 }, "aisimulate": { "tpot_error_pct": 16.19, "tpot_relative": 1.135411, "ttft_error_pct": 62.6, "ttft_relative": 0.757728 }, "concurrency": 128, "measured": { "tpot_relative": 1.354673, "ttft_relative": 2.026276 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 16.8, "tpot_shape_error_pct": 21.27, "ttft_mape_pct": 58.43, "ttft_shape_error_pct": 55.43 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.51, "tpot_shape_error_pct": 21.84, "ttft_mape_pct": 66.48, "ttft_shape_error_pct": 28.12 }, "framework": "trtllm", "id": "1bda030db398071e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 12.68, "tpot_relative": 0.873204, "ttft_error_pct": 28.85, "ttft_relative": 0.711492 }, "aisimulate": { "tpot_error_pct": 13.72, "tpot_relative": 0.862771, "ttft_error_pct": 57.52, "ttft_relative": 0.424761 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.65, "tpot_relative": 1.15534, "ttft_error_pct": 47.03, "ttft_relative": 0.836448 }, "aisimulate": { "tpot_error_pct": 7.78, "tpot_relative": 1.14137, "ttft_error_pct": 62.2, "ttft_relative": 0.597008 }, "concurrency": 64, "measured": { "tpot_relative": 1.237693, "ttft_relative": 1.579243 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.76, "tpot_relative": 1.478261, "ttft_error_pct": 68.59, "ttft_relative": 0.859534 }, "aisimulate": { "tpot_error_pct": 12.69, "tpot_relative": 1.462734, "ttft_error_pct": 65.9, "ttft_relative": 0.933052 }, "concurrency": 128, "measured": { "tpot_relative": 1.675274, "ttft_relative": 2.736375 }, "status": "success" }, { "aic": { "tpot_error_pct": 36.1, "tpot_relative": 2.623138, "ttft_error_pct": 89.25, "ttft_relative": 0.906421 }, "aisimulate": { "tpot_error_pct": 35.83, "tpot_relative": 2.617912, "ttft_error_pct": 80.31, "ttft_relative": 1.660755 }, "concurrency": 256, "measured": { "tpot_relative": 1.927324, "ttft_relative": 8.433154 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:8192", "label": "1k8k", "precisions": [ "fp4" ], "rows": 12 }, { "aic": { "points": 39, "tpot_mape_pct": 21.98, "tpot_shape_error_pct": 13.84, "ttft_mape_pct": 40.03, "ttft_shape_error_pct": 41.56 }, "aisimulate": { "coverage_pct": 89.74, "points": 35, "status_counts": { "failed": 4, "success": 35, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 24.05, "tpot_shape_error_pct": 13.8, "ttft_mape_pct": 41.09, "ttft_shape_error_pct": 28.53 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 25, "tpot_mape_pct": 14.24, "tpot_shape_error_pct": 11.34, "ttft_mape_pct": 45.34, "ttft_shape_error_pct": 47.03 }, "aisimulate": { "coverage_pct": 84.0, "points": 21, "status_counts": { "failed": 4, "success": 21, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.62, "tpot_shape_error_pct": 12.37, "ttft_mape_pct": 42.12, "ttft_shape_error_pct": 33.97 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 25, "topologies": [ { "aic": { "points": 8, "tpot_mape_pct": 10.72, "tpot_shape_error_pct": 11.17, "ttft_mape_pct": 42.96, "ttft_shape_error_pct": 54.35 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.06, "tpot_shape_error_pct": 11.08, "ttft_mape_pct": 58.97, "ttft_shape_error_pct": 46.14 }, "framework": "sglang", "id": "bc878ae77b0d27c7", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 6.93, "tpot_relative": 1.069314, "ttft_error_pct": 43.76, "ttft_relative": 1.437615 }, "aisimulate": { "tpot_error_pct": 6.49, "tpot_relative": 1.064901, "ttft_error_pct": 31.18, "ttft_relative": 0.688169 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.16, "tpot_relative": 1.196587, "ttft_error_pct": 21.79, "ttft_relative": 1.475633 }, "aisimulate": { "tpot_error_pct": 3.57, "tpot_relative": 1.189874, "ttft_error_pct": 41.02, "ttft_relative": 0.714673 }, "concurrency": 2, "measured": { "tpot_relative": 1.148836, "ttft_relative": 1.211644 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.7, "tpot_relative": 1.556068, "ttft_error_pct": 7.87, "ttft_relative": 1.551697 }, "aisimulate": { "tpot_error_pct": 18.57, "tpot_relative": 1.594647, "ttft_error_pct": 45.08, "ttft_relative": 0.790035 }, "concurrency": 4, "measured": { "tpot_relative": 1.344895, "ttft_relative": 1.438515 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.05, "tpot_relative": 1.831018, "ttft_error_pct": 4.83, "ttft_relative": 1.703939 }, "aisimulate": { "tpot_error_pct": 9.01, "tpot_relative": 1.864534, "ttft_error_pct": 47.51, "ttft_relative": 0.939805 }, "concurrency": 8, "measured": { "tpot_relative": 1.710385, "ttft_relative": 1.790351 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.76, "tpot_relative": 2.45657, "ttft_error_pct": 32.64, "ttft_relative": 2.008881 }, "aisimulate": { "tpot_error_pct": 4.94, "tpot_relative": 2.484556, "ttft_error_pct": 59.67, "ttft_relative": 1.202809 }, "concurrency": 16, "measured": { "tpot_relative": 2.367615, "ttft_relative": 2.982155 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.06, "tpot_relative": 3.506219, "ttft_error_pct": 45.06, "ttft_relative": 2.620596 }, "aisimulate": { "tpot_error_pct": 0.91, "tpot_relative": 3.535807, "ttft_error_pct": 62.71, "ttft_relative": 1.778603 }, "concurrency": 32, "measured": { "tpot_relative": 3.503966, "ttft_relative": 4.770289 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.57, "tpot_relative": 5.330742, "ttft_error_pct": 92.99, "ttft_relative": 3.05058 }, "aisimulate": { "tpot_error_pct": 22.56, "tpot_relative": 5.401176, "ttft_error_pct": 93.35, "ttft_relative": 2.893731 }, "concurrency": 64, "measured": { "tpot_relative": 6.97506, "ttft_relative": 43.537532 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.5, "tpot_relative": 8.063834, "ttft_error_pct": 94.73, "ttft_relative": 3.075027 }, "aisimulate": { "tpot_error_pct": 22.4, "tpot_relative": 8.288162, "ttft_error_pct": 91.23, "ttft_relative": 5.113952 }, "concurrency": 128, "measured": { "tpot_relative": 10.680775, "ttft_relative": 58.31382 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 5.79, "tpot_shape_error_pct": 3.1, "ttft_mape_pct": 11.3, "ttft_shape_error_pct": 21.51 }, "aisimulate": { "points": 0, "status_counts": { "failed": 4, "success": 0, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "868c7e6f6888ecd7", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 3.54, "tpot_relative": 0.964558, "ttft_error_pct": 17.04, "ttft_relative": 1.170356 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "failed" }, { "aic": { "tpot_error_pct": 6.7, "tpot_relative": 1.070203, "ttft_error_pct": 1.88, "ttft_relative": 1.201296 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 2, "measured": { "tpot_relative": 1.147057, "ttft_relative": 1.179094 }, "status": "failed" }, { "aic": { "tpot_error_pct": 6.42, "tpot_relative": 1.338356, "ttft_error_pct": 6.61, "ttft_relative": 1.263198 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 4, "measured": { "tpot_relative": 1.430193, "ttft_relative": 1.352614 }, "status": "failed" }, { "aic": { "tpot_error_pct": 6.48, "tpot_relative": 1.769862, "ttft_error_pct": 19.69, "ttft_relative": 1.387089 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 8, "measured": { "tpot_relative": 1.892515, "ttft_relative": 1.727075 }, "status": "failed" } ], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 8.33, "tpot_shape_error_pct": 13.26, "ttft_mape_pct": 9.94, "ttft_shape_error_pct": 18.46 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.24, "tpot_shape_error_pct": 11.57, "ttft_mape_pct": 51.67, "ttft_shape_error_pct": 15.45 }, "framework": "sglang", "id": "8fb99eea6114ceee", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 2.09, "tpot_relative": 1.020882, "ttft_error_pct": 11.22, "ttft_relative": 1.112182 }, "aisimulate": { "tpot_error_pct": 1.84, "tpot_relative": 1.018432, "ttft_error_pct": 46.12, "ttft_relative": 0.538831 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.69, "tpot_relative": 0.996497, "ttft_error_pct": 3.31, "ttft_relative": 1.141569 }, "aisimulate": { "tpot_error_pct": 10.83, "tpot_relative": 0.994886, "ttft_error_pct": 52.54, "ttft_relative": 0.56038 }, "concurrency": 2, "measured": { "tpot_relative": 1.11574, "ttft_relative": 1.180675 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.21, "tpot_relative": 1.126277, "ttft_error_pct": 15.3, "ttft_relative": 1.200361 }, "aisimulate": { "tpot_error_pct": 9.05, "tpot_relative": 1.166803, "ttft_error_pct": 56.34, "ttft_relative": 0.618722 }, "concurrency": 4, "measured": { "tpot_relative": 1.282856, "ttft_relative": 1.417236 }, "status": "success" } ], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 15.55, "tpot_shape_error_pct": 5.36, "ttft_mape_pct": 70.3, "ttft_shape_error_pct": 41.25 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.98, "tpot_shape_error_pct": 3.87, "ttft_mape_pct": 19.94, "ttft_shape_error_pct": 19.3 }, "framework": "trtllm", "id": "7f0e371bf1187736", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 11.86, "tpot_relative": 1.118644, "ttft_error_pct": 102.11, "ttft_relative": 2.021061 }, "aisimulate": { "tpot_error_pct": 16.97, "tpot_relative": 1.169727, "ttft_error_pct": 7.82, "ttft_relative": 1.078155 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.57, "tpot_relative": 1.417633, "ttft_error_pct": 84.29, "ttft_relative": 2.219216 }, "aisimulate": { "tpot_error_pct": 18.76, "tpot_relative": 1.456735, "ttft_error_pct": 10.86, "ttft_relative": 1.334904 }, "concurrency": 8, "measured": { "tpot_relative": 1.226635, "ttft_relative": 1.204171 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.8, "tpot_relative": 1.967135, "ttft_error_pct": 69.2, "ttft_relative": 2.616048 }, "aisimulate": { "tpot_error_pct": 16.79, "tpot_relative": 2.001166, "ttft_error_pct": 10.1, "ttft_relative": 1.702322 }, "concurrency": 16, "measured": { "tpot_relative": 1.713472, "ttft_relative": 1.546092 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.59, "tpot_relative": 2.875539, "ttft_error_pct": 53.2, "ttft_relative": 3.411803 }, "aisimulate": { "tpot_error_pct": 16.82, "tpot_relative": 2.906182, "ttft_error_pct": 12.44, "ttft_relative": 2.503992 }, "concurrency": 32, "measured": { "tpot_relative": 2.487684, "ttft_relative": 2.226994 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.93, "tpot_relative": 4.427318, "ttft_error_pct": 50.18, "ttft_relative": 3.969645 }, "aisimulate": { "tpot_error_pct": 9.01, "tpot_relative": 4.47169, "ttft_error_pct": 48.28, "ttft_relative": 4.12049 }, "concurrency": 64, "measured": { "tpot_relative": 4.102024, "ttft_relative": 7.967464 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.53, "tpot_relative": 7.027593, "ttft_error_pct": 62.84, "ttft_relative": 3.997539 }, "aisimulate": { "tpot_error_pct": 29.52, "tpot_relative": 7.137683, "ttft_error_pct": 30.16, "ttft_relative": 7.513572 }, "concurrency": 128, "measured": { "tpot_relative": 5.510684, "ttft_relative": 10.757606 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 32.24, "tpot_shape_error_pct": 28.64, "ttft_mape_pct": 73.26, "ttft_shape_error_pct": 84.13 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 38.0, "tpot_shape_error_pct": 30.07, "ttft_mape_pct": 34.5, "ttft_shape_error_pct": 42.39 }, "framework": "trtllm", "id": "063c0af5eee83fde", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 10.59, "tpot_relative": 1.105915, "ttft_error_pct": 77.57, "ttft_relative": 1.775682 }, "aisimulate": { "tpot_error_pct": 15.69, "tpot_relative": 1.156862, "ttft_error_pct": 3.98, "ttft_relative": 0.960225 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.37, "tpot_relative": 4.004206, "ttft_error_pct": 64.37, "ttft_relative": 3.485823 }, "aisimulate": { "tpot_error_pct": 8.12, "tpot_relative": 4.070161, "ttft_error_pct": 62.96, "ttft_relative": 3.62393 }, "concurrency": 64, "measured": { "tpot_relative": 3.764471, "ttft_relative": 9.782991 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.08, "tpot_relative": 6.342838, "ttft_error_pct": 71.8, "ttft_relative": 3.508334 }, "aisimulate": { "tpot_error_pct": 38.28, "tpot_relative": 6.492894, "ttft_error_pct": 46.96, "ttft_relative": 6.597219 }, "concurrency": 128, "measured": { "tpot_relative": 4.69558, "ttft_relative": 12.438873 }, "status": "success" }, { "aic": { "tpot_error_pct": 76.9, "tpot_relative": 11.583691, "ttft_error_pct": 79.32, "ttft_relative": 3.553436 }, "aisimulate": { "tpot_error_pct": 89.91, "tpot_relative": 12.43505, "ttft_error_pct": 24.12, "ttft_relative": 13.037983 }, "concurrency": 256, "measured": { "tpot_relative": 6.548006, "ttft_relative": 17.181729 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 9, "tpot_mape_pct": 9.92, "tpot_shape_error_pct": 3.79, "ttft_mape_pct": 17.25, "ttft_shape_error_pct": 28.38 }, "aisimulate": { "coverage_pct": 100.0, "points": 9, "status_counts": { "failed": 0, "success": 9, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.84, "tpot_shape_error_pct": 2.98, "ttft_mape_pct": 52.76, "ttft_shape_error_pct": 20.13 }, "gpu": "b300", "precisions": [ "fp8" ], "rows": 9, "topologies": [ { "aic": { "points": 3, "tpot_mape_pct": 9.21, "tpot_shape_error_pct": 4.12, "ttft_mape_pct": 10.78, "ttft_shape_error_pct": 20.46 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.35, "tpot_shape_error_pct": 2.44, "ttft_mape_pct": 50.83, "ttft_shape_error_pct": 17.49 }, "framework": "sglang", "id": "d3b16e4bdf5ac2f7", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 6.64, "tpot_relative": 0.933562, "ttft_error_pct": 14.53, "ttft_relative": 1.145311 }, "aisimulate": { "tpot_error_pct": 6.89, "tpot_relative": 0.931072, "ttft_error_pct": 44.34, "ttft_relative": 0.556607 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.07, "tpot_relative": 0.994227, "ttft_error_pct": 3.56, "ttft_relative": 1.175572 }, "aisimulate": { "tpot_error_pct": 11.34, "tpot_relative": 0.991203, "ttft_error_pct": 52.49, "ttft_relative": 0.57908 }, "concurrency": 2, "measured": { "tpot_relative": 1.118039, "ttft_relative": 1.218938 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.9, "tpot_relative": 1.141875, "ttft_error_pct": 14.25, "ttft_relative": 1.236113 }, "aisimulate": { "tpot_error_pct": 6.8, "tpot_relative": 1.181139, "ttft_error_pct": 55.66, "ttft_relative": 0.63917 }, "concurrency": 4, "measured": { "tpot_relative": 1.267324, "ttft_relative": 1.441542 }, "status": "success" } ], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 10.27, "tpot_shape_error_pct": 3.66, "ttft_mape_pct": 20.49, "ttft_shape_error_pct": 31.55 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.09, "tpot_shape_error_pct": 3.19, "ttft_mape_pct": 53.72, "ttft_shape_error_pct": 21.18 }, "framework": "sglang", "id": "eccdd51bc4f9ab1a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 9.59, "tpot_relative": 0.904121, "ttft_error_pct": 16.76, "ttft_relative": 1.167636 }, "aisimulate": { "tpot_error_pct": 9.76, "tpot_relative": 0.902381, "ttft_error_pct": 43.8, "ttft_relative": 0.562003 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.66, "tpot_relative": 1.090113, "ttft_error_pct": 2.89, "ttft_relative": 1.198505 }, "aisimulate": { "tpot_error_pct": 5.25, "tpot_relative": 1.08344, "ttft_error_pct": 49.86, "ttft_relative": 0.584015 }, "concurrency": 2, "measured": { "tpot_relative": 1.143455, "ttft_relative": 1.164786 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.29, "tpot_relative": 1.313461, "ttft_error_pct": 8.7, "ttft_relative": 1.260264 }, "aisimulate": { "tpot_error_pct": 5.06, "tpot_relative": 1.359594, "ttft_error_pct": 53.26, "ttft_relative": 0.645251 }, "concurrency": 4, "measured": { "tpot_relative": 1.432124, "ttft_relative": 1.380364 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.08, "tpot_relative": 1.675035, "ttft_error_pct": 19.85, "ttft_relative": 1.383868 }, "aisimulate": { "tpot_error_pct": 9.87, "tpot_relative": 1.717153, "ttft_error_pct": 55.53, "ttft_relative": 0.767786 }, "concurrency": 8, "measured": { "tpot_relative": 1.905154, "ttft_relative": 1.726647 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.2, "tpot_relative": 2.441989, "ttft_error_pct": 32.9, "ttft_relative": 1.631425 }, "aisimulate": { "tpot_error_pct": 9.88, "tpot_relative": 2.478496, "ttft_error_pct": 59.59, "ttft_relative": 0.98255 }, "concurrency": 16, "measured": { "tpot_relative": 2.750124, "ttft_relative": 2.431441 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.8, "tpot_relative": 3.504228, "ttft_error_pct": 41.83, "ttft_relative": 2.127932 }, "aisimulate": { "tpot_error_pct": 14.72, "tpot_relative": 3.549337, "ttft_error_pct": 60.28, "ttft_relative": 1.452986 }, "concurrency": 32, "measured": { "tpot_relative": 4.161897, "ttft_relative": 3.657964 }, "status": "success" } ], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 5, "tpot_mape_pct": 82.34, "tpot_shape_error_pct": 43.92, "ttft_mape_pct": 54.49, "ttft_shape_error_pct": 37.27 }, "aisimulate": { "coverage_pct": 100.0, "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 78.45, "tpot_shape_error_pct": 38.83, "ttft_mape_pct": 15.78, "ttft_shape_error_pct": 20.1 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 5, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 82.34, "tpot_shape_error_pct": 43.92, "ttft_mape_pct": 54.49, "ttft_shape_error_pct": 37.27 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 78.45, "tpot_shape_error_pct": 38.83, "ttft_mape_pct": 15.78, "ttft_shape_error_pct": 20.1 }, "framework": "sglang", "id": "d9fd8e0fd2504827", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 34.93, "tpot_relative": 1.349317, "ttft_error_pct": 100.51, "ttft_relative": 2.00511 }, "aisimulate": { "tpot_error_pct": 36.15, "tpot_relative": 1.361502, "ttft_error_pct": 0.68, "ttft_relative": 1.006831 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 59.96, "tpot_relative": 2.029238, "ttft_error_pct": 67.56, "ttft_relative": 2.202031 }, "aisimulate": { "tpot_error_pct": 57.81, "tpot_relative": 2.001933, "ttft_error_pct": 8.93, "ttft_relative": 1.196861 }, "concurrency": 8, "measured": { "tpot_relative": 1.268604, "ttft_relative": 1.314152 }, "status": "success" }, { "aic": { "tpot_error_pct": 92.24, "tpot_relative": 3.517716, "ttft_error_pct": 40.83, "ttft_relative": 2.596569 }, "aisimulate": { "tpot_error_pct": 86.77, "tpot_relative": 3.417582, "ttft_error_pct": 16.9, "ttft_relative": 1.532204 }, "concurrency": 16, "measured": { "tpot_relative": 1.829858, "ttft_relative": 1.843744 }, "status": "success" }, { "aic": { "tpot_error_pct": 103.2, "tpot_relative": 5.82816, "ttft_error_pct": 29.15, "ttft_relative": 3.388426 }, "aisimulate": { "tpot_error_pct": 96.21, "tpot_relative": 5.627603, "ttft_error_pct": 13.66, "ttft_relative": 2.265352 }, "concurrency": 32, "measured": { "tpot_relative": 2.86816, "ttft_relative": 2.623724 }, "status": "success" }, { "aic": { "tpot_error_pct": 121.39, "tpot_relative": 9.984601, "ttft_error_pct": 34.39, "ttft_relative": 3.947158 }, "aisimulate": { "tpot_error_pct": 115.31, "tpot_relative": 9.710577, "ttft_error_pct": 38.74, "ttft_relative": 3.685542 }, "concurrency": 64, "measured": { "tpot_relative": 4.509982, "ttft_relative": 6.016042 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4", "fp8" ], "rows": 39 } ] }, { "aic": { "points": 68, "tpot_mape_pct": 80.72, "tpot_shape_error_pct": 33.05, "ttft_mape_pct": 48.59, "ttft_shape_error_pct": 42.45 }, "aisimulate": { "coverage_pct": 52.94, "points": 36, "status_counts": { "failed": 14, "success": 36, "unknown": 0, "unsupported": 18 }, "tpot_mape_pct": 73.42, "tpot_shape_error_pct": 22.01, "ttft_mape_pct": 50.78, "ttft_shape_error_pct": 46.55 }, "frameworks": [ "dynamo-sglang", "sglang", "vllm" ], "gpu_skus": [ "b200", "b300" ], "hf_model_paths": [ "deepseek-ai/DeepSeek-V4-Pro" ], "model": "DeepSeek-V4-Pro", "precisions": [ "fp4" ], "rows": 68, "workloads": [ { "aic": { "points": 27, "tpot_mape_pct": 70.23, "tpot_shape_error_pct": 26.66, "ttft_mape_pct": 72.28, "ttft_shape_error_pct": 35.39 }, "aisimulate": { "coverage_pct": 88.89, "points": 24, "status_counts": { "failed": 3, "success": 24, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 80.22, "tpot_shape_error_pct": 26.23, "ttft_mape_pct": 55.68, "ttft_shape_error_pct": 33.19 }, "gpu_skus": [ "b200", "b300" ], "gpus": [ { "aic": { "points": 6, "tpot_mape_pct": 14.63, "tpot_shape_error_pct": 5.91, "ttft_mape_pct": 70.3, "ttft_shape_error_pct": 47.27 }, "aisimulate": { "coverage_pct": 100.0, "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.19, "tpot_shape_error_pct": 5.24, "ttft_mape_pct": 85.81, "ttft_shape_error_pct": 44.13 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 6, "topologies": [ { "aic": { "points": 6, "tpot_mape_pct": 14.63, "tpot_shape_error_pct": 5.91, "ttft_mape_pct": 70.3, "ttft_shape_error_pct": 47.27 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.19, "tpot_shape_error_pct": 5.24, "ttft_mape_pct": 85.81, "ttft_shape_error_pct": 44.13 }, "framework": "sglang", "id": "fdf4e0bfefe7924c", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 10.2, "tpot_relative": 0.89799, "ttft_error_pct": 78.09, "ttft_relative": 0.219129 }, "aisimulate": { "tpot_error_pct": 10.27, "tpot_relative": 0.89731, "ttft_error_pct": 89.21, "ttft_relative": 0.107929 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.33, "tpot_relative": 0.97207, "ttft_error_pct": 80.62, "ttft_relative": 0.238609 }, "aisimulate": { "tpot_error_pct": 15.63, "tpot_relative": 0.968601, "ttft_error_pct": 90.92, "ttft_relative": 0.111832 }, "concurrency": 2, "measured": { "tpot_relative": 1.148054, "ttft_relative": 1.231448 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.5, "tpot_relative": 1.057219, "ttft_error_pct": 71.88, "ttft_relative": 0.252816 }, "aisimulate": { "tpot_error_pct": 15.74, "tpot_relative": 1.066771, "ttft_error_pct": 86.41, "ttft_relative": 0.122178 }, "concurrency": 4, "measured": { "tpot_relative": 1.266109, "ttft_relative": 0.898909 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.99, "tpot_relative": 1.227842, "ttft_error_pct": 66.43, "ttft_relative": 0.278126 }, "aisimulate": { "tpot_error_pct": 15.44, "tpot_relative": 1.235963, "ttft_error_pct": 83.41, "ttft_relative": 0.137425 }, "concurrency": 8, "measured": { "tpot_relative": 1.461594, "ttft_relative": 0.828571 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.85, "tpot_relative": 1.535922, "ttft_error_pct": 71.35, "ttft_relative": 0.328447 }, "aisimulate": { "tpot_error_pct": 13.09, "tpot_relative": 1.549375, "ttft_error_pct": 86.39, "ttft_relative": 0.15601 }, "concurrency": 16, "measured": { "tpot_relative": 1.782749, "ttft_relative": 1.146501 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.88, "tpot_relative": 1.955776, "ttft_error_pct": 53.43, "ttft_relative": 0.430651 }, "aisimulate": { "tpot_error_pct": 14.97, "tpot_relative": 1.976948, "ttft_error_pct": 78.51, "ttft_relative": 0.198719 }, "concurrency": 32, "measured": { "tpot_relative": 2.325091, "ttft_relative": 0.924799 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 21, "tpot_mape_pct": 86.11, "tpot_shape_error_pct": 34.64, "ttft_mape_pct": 72.85, "ttft_shape_error_pct": 30.82 }, "aisimulate": { "coverage_pct": 85.71, "points": 18, "status_counts": { "failed": 3, "success": 18, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 102.23, "tpot_shape_error_pct": 34.98, "ttft_mape_pct": 45.64, "ttft_shape_error_pct": 28.64 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 21, "topologies": [ { "aic": { "points": 1, "tpot_mape_pct": 17.95, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.53, "ttft_shape_error_pct": null }, "aisimulate": { "points": 0, "status_counts": { "failed": 1, "success": 0, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "07be42110e785c45", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 17.95, "tpot_relative": 0.820473, "ttft_error_pct": 43.53, "ttft_relative": 0.564731 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 512, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "failed" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 11.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 87.57, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.74, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.87, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "ce2ca8bcfa14d009", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 11.42, "tpot_relative": 0.885764, "ttft_error_pct": 87.57, "ttft_relative": 0.124263 }, "aisimulate": { "tpot_error_pct": 11.74, "tpot_relative": 0.882554, "ttft_error_pct": 93.87, "ttft_relative": 0.061288 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 3.76, "tpot_shape_error_pct": null, "ttft_mape_pct": 17.89, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.92, "tpot_shape_error_pct": null, "ttft_mape_pct": 64.06, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "aa9317c313198ad9", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 3.76, "tpot_relative": 1.037563, "ttft_error_pct": 17.89, "ttft_relative": 0.821109 }, "aisimulate": { "tpot_error_pct": 4.92, "tpot_relative": 1.04918, "ttft_error_pct": 64.06, "ttft_relative": 0.359353 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 207.24, "tpot_shape_error_pct": 24.45, "ttft_mape_pct": 99.08, "ttft_shape_error_pct": 2.51 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 208.21, "tpot_shape_error_pct": 24.06, "ttft_mape_pct": 52.16, "ttft_shape_error_pct": 22.4 }, "framework": "vllm", "id": "82b99448af6ea388", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 267.06, "tpot_relative": 3.670635, "ttft_error_pct": 98.03, "ttft_relative": 1.980258 }, "aisimulate": { "tpot_error_pct": 267.09, "tpot_relative": 3.670877, "ttft_error_pct": 78.87, "ttft_relative": 1.788706 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 214.95, "tpot_relative": 3.308942, "ttft_error_pct": 94.64, "ttft_relative": 2.43005 }, "aisimulate": { "tpot_error_pct": 215.53, "tpot_relative": 3.315026, "ttft_error_pct": 41.72, "ttft_relative": 1.769371 }, "concurrency": 2, "measured": { "tpot_relative": 1.050633, "ttft_relative": 1.24847 }, "status": "success" }, { "aic": { "tpot_error_pct": 139.72, "tpot_relative": 2.758935, "ttft_error_pct": 104.57, "ttft_relative": 2.699884 }, "aisimulate": { "tpot_error_pct": 142.01, "tpot_relative": 2.785306, "ttft_error_pct": 35.88, "ttft_relative": 1.793338 }, "concurrency": 4, "measured": { "tpot_relative": 1.150914, "ttft_relative": 1.31976 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 30.22, "tpot_shape_error_pct": 5.43, "ttft_mape_pct": 55.68, "ttft_shape_error_pct": 31.1 }, "aisimulate": { "points": 0, "status_counts": { "failed": 2, "success": 0, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "13c32ca95b95cbb2", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 32.07, "tpot_relative": 0.679311, "ttft_error_pct": 47.52, "ttft_relative": 0.524794 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "failed" }, { "aic": { "tpot_error_pct": 28.38, "tpot_relative": 0.950283, "ttft_error_pct": 63.84, "ttft_relative": 0.564604 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 512, "measured": { "tpot_relative": 1.326822, "ttft_relative": 1.561501 }, "status": "failed" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 8, "tpot_mape_pct": 126.31, "tpot_shape_error_pct": 49.06, "ttft_mape_pct": 74.41, "ttft_shape_error_pct": 23.91 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 128.43, "tpot_shape_error_pct": 48.45, "ttft_mape_pct": 49.27, "ttft_shape_error_pct": 28.18 }, "framework": "vllm", "id": "67bfeaf1bd4f5fde", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 296.51, "tpot_relative": 3.965076, "ttft_error_pct": 109.99, "ttft_relative": 2.099936 }, "aisimulate": { "tpot_error_pct": 296.54, "tpot_relative": 3.965354, "ttft_error_pct": 98.13, "ttft_relative": 1.981251 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 225.02, "tpot_relative": 3.433909, "ttft_error_pct": 105.46, "ttft_relative": 2.550302 }, "aisimulate": { "tpot_error_pct": 225.7, "tpot_relative": 3.441157, "ttft_error_pct": 55.9, "ttft_relative": 1.935224 }, "concurrency": 2, "measured": { "tpot_relative": 1.056533, "ttft_relative": 1.24129 }, "status": "success" }, { "aic": { "tpot_error_pct": 149.94, "tpot_relative": 2.919878, "ttft_error_pct": 110.2, "ttft_relative": 2.836815 }, "aisimulate": { "tpot_error_pct": 149.92, "tpot_relative": 2.919741, "ttft_error_pct": 46.16, "ttft_relative": 1.972523 }, "concurrency": 4, "measured": { "tpot_relative": 1.168252, "ttft_relative": 1.349568 }, "status": "success" }, { "aic": { "tpot_error_pct": 118.69, "tpot_relative": 2.89553, "ttft_error_pct": 104.0, "ttft_relative": 3.119213 }, "aisimulate": { "tpot_error_pct": 120.58, "tpot_relative": 2.920547, "ttft_error_pct": 43.47, "ttft_relative": 2.193742 }, "concurrency": 8, "measured": { "tpot_relative": 1.324037, "ttft_relative": 1.529041 }, "status": "success" }, { "aic": { "tpot_error_pct": 95.11, "tpot_relative": 3.033984, "ttft_error_pct": 81.97, "ttft_relative": 3.397207 }, "aisimulate": { "tpot_error_pct": 97.64, "tpot_relative": 3.073407, "ttft_error_pct": 40.51, "ttft_relative": 2.62329 }, "concurrency": 16, "measured": { "tpot_relative": 1.555028, "ttft_relative": 1.866917 }, "status": "success" }, { "aic": { "tpot_error_pct": 61.45, "tpot_relative": 3.16814, "ttft_error_pct": 46.3, "ttft_relative": 3.674169 }, "aisimulate": { "tpot_error_pct": 63.99, "tpot_relative": 3.218032, "ttft_error_pct": 37.36, "ttft_relative": 3.449481 }, "concurrency": 32, "measured": { "tpot_relative": 1.962355, "ttft_relative": 2.511356 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.41, "tpot_relative": 3.552544, "ttft_error_pct": 4.16, "ttft_relative": 3.935824 }, "aisimulate": { "tpot_error_pct": 39.18, "tpot_relative": 3.651555, "ttft_error_pct": 35.36, "ttft_relative": 5.114612 }, "concurrency": 64, "measured": { "tpot_relative": 2.62356, "ttft_relative": 3.778469 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.37, "tpot_relative": 4.472556, "ttft_error_pct": 33.17, "ttft_relative": 4.177749 }, "aisimulate": { "tpot_error_pct": 33.9, "tpot_relative": 4.665042, "ttft_error_pct": 37.25, "ttft_relative": 8.58051 }, "concurrency": 128, "measured": { "tpot_relative": 3.48409, "ttft_relative": 6.251586 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 13.61, "tpot_shape_error_pct": 15.42, "ttft_mape_pct": 64.38, "ttft_shape_error_pct": 64.68 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 27.58, "tpot_shape_error_pct": 14.3, "ttft_mape_pct": 31.4, "ttft_shape_error_pct": 46.92 }, "framework": "vllm", "id": "238f245c0ef0160a", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 14.45, "tpot_relative": 0.855497, "ttft_error_pct": 37.37, "ttft_relative": 0.626286 }, "aisimulate": { "tpot_error_pct": 16.48, "tpot_relative": 1.164801, "ttft_error_pct": 5.96, "ttft_relative": 0.940352 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.5, "tpot_relative": 1.105745, "ttft_error_pct": 57.32, "ttft_relative": 0.680744 }, "aisimulate": { "tpot_error_pct": 20.74, "tpot_relative": 1.579928, "ttft_error_pct": 0.63, "ttft_relative": 1.60492 }, "concurrency": 512, "measured": { "tpot_relative": 1.308558, "ttft_relative": 1.594904 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.89, "tpot_relative": 2.181461, "ttft_error_pct": 98.44, "ttft_relative": 0.80654 }, "aisimulate": { "tpot_error_pct": 45.54, "tpot_relative": 2.863009, "ttft_error_pct": 87.61, "ttft_relative": 6.421191 }, "concurrency": 2048, "measured": { "tpot_relative": 1.967225, "ttft_relative": 51.827237 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 20.86, "tpot_shape_error_pct": 21.79, "ttft_mape_pct": 91.95, "ttft_shape_error_pct": 67.76 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 44.31, "tpot_shape_error_pct": 3.87, "ttft_mape_pct": 9.41, "ttft_shape_error_pct": 7.74 }, "framework": "vllm", "id": "cc02b368d685079f", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 8.99, "tpot_relative": 1.089869, "ttft_error_pct": 87.82, "ttft_relative": 0.121795 }, "aisimulate": { "tpot_error_pct": 47.15, "tpot_relative": 1.471532, "ttft_error_pct": 5.76, "ttft_relative": 0.942389 }, "concurrency": 4096, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 32.74, "tpot_relative": 1.977916, "ttft_error_pct": 96.07, "ttft_relative": 0.148783 }, "aisimulate": { "tpot_error_pct": 41.46, "tpot_relative": 2.107938, "ttft_error_pct": 13.06, "ttft_relative": 3.294745 }, "concurrency": 8192, "measured": { "tpot_relative": 1.490087, "ttft_relative": 3.789552 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4" ], "rows": 27 }, { "aic": { "points": 41, "tpot_mape_pct": 87.63, "tpot_shape_error_pct": 36.65, "ttft_mape_pct": 32.99, "ttft_shape_error_pct": 46.43 }, "aisimulate": { "coverage_pct": 29.27, "points": 12, "status_counts": { "failed": 11, "success": 12, "unknown": 0, "unsupported": 18 }, "tpot_mape_pct": 59.83, "tpot_shape_error_pct": 11.75, "ttft_mape_pct": 40.97, "ttft_shape_error_pct": 78.99 }, "gpu_skus": [ "b200", "b300" ], "gpus": [ { "aic": { "points": 24, "tpot_mape_pct": 80.42, "tpot_shape_error_pct": 35.26, "ttft_mape_pct": 31.64, "ttft_shape_error_pct": 50.23 }, "aisimulate": { "coverage_pct": 25.0, "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 18 }, "tpot_mape_pct": 8.58, "tpot_shape_error_pct": 6.22, "ttft_mape_pct": 47.79, "ttft_shape_error_pct": 99.88 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 24, "topologies": [ { "aic": { "points": 6, "tpot_mape_pct": 9.78, "tpot_shape_error_pct": 6.58, "ttft_mape_pct": 17.04, "ttft_shape_error_pct": 72.68 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.58, "tpot_shape_error_pct": 6.22, "ttft_mape_pct": 47.79, "ttft_shape_error_pct": 99.88 }, "framework": "sglang", "id": "bce9e8c274778f29", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 9.86, "tpot_relative": 0.901404, "ttft_error_pct": 41.22, "ttft_relative": 0.58777 }, "aisimulate": { "tpot_error_pct": 9.96, "tpot_relative": 0.90045, "ttft_error_pct": 71.51, "ttft_relative": 0.284929 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.4, "tpot_relative": 1.001796, "ttft_error_pct": 14.92, "ttft_relative": 0.612156 }, "aisimulate": { "tpot_error_pct": 13.58, "tpot_relative": 0.999774, "ttft_error_pct": 58.81, "ttft_relative": 0.296333 }, "concurrency": 2, "measured": { "tpot_relative": 1.156828, "ttft_relative": 0.719484 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.28, "tpot_relative": 1.082609, "ttft_error_pct": 18.13, "ttft_relative": 0.644965 }, "aisimulate": { "tpot_error_pct": 13.4, "tpot_relative": 1.133452, "ttft_error_pct": 40.07, "ttft_relative": 0.327173 }, "concurrency": 4, "measured": { "tpot_relative": 1.308795, "ttft_relative": 0.545957 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.46, "tpot_relative": 1.333265, "ttft_error_pct": 3.6, "ttft_relative": 0.708329 }, "aisimulate": { "tpot_error_pct": 9.49, "tpot_relative": 1.378432, "ttft_error_pct": 43.04, "ttft_relative": 0.389449 }, "concurrency": 8, "measured": { "tpot_relative": 1.523017, "ttft_relative": 0.683738 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.65, "tpot_relative": 1.819931, "ttft_error_pct": 12.52, "ttft_relative": 0.835345 }, "aisimulate": { "tpot_error_pct": 2.26, "tpot_relative": 1.865516, "ttft_error_pct": 32.88, "ttft_relative": 0.498315 }, "concurrency": 16, "measured": { "tpot_relative": 1.908615, "ttft_relative": 0.742412 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.01, "tpot_relative": 2.58378, "ttft_error_pct": 11.86, "ttft_relative": 1.090432 }, "aisimulate": { "tpot_error_pct": 2.81, "tpot_relative": 2.629804, "ttft_error_pct": 40.43, "ttft_relative": 0.736961 }, "concurrency": 32, "measured": { "tpot_relative": 2.557926, "ttft_relative": 1.237116 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 9, "tpot_mape_pct": 111.58, "tpot_shape_error_pct": 44.76, "ttft_mape_pct": 37.2, "ttft_shape_error_pct": 45.41 }, "aisimulate": { "points": 0, "status_counts": { "failed": 0, "success": 0, "unknown": 0, "unsupported": 9 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "f104d08d7d025a3e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 251.37, "tpot_relative": 3.51372, "ttft_error_pct": 10.81, "ttft_relative": 0.891917 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 227.73, "tpot_relative": 3.520904, "ttft_error_pct": 14.33, "ttft_relative": 1.014189 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 2, "measured": { "tpot_relative": 1.074325, "ttft_relative": 0.887075 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 142.79, "tpot_relative": 2.852688, "ttft_error_pct": 55.45, "ttft_relative": 1.127382 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 4, "measured": { "tpot_relative": 1.174949, "ttft_relative": 0.725236 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 111.76, "tpot_relative": 3.010213, "ttft_error_pct": 28.46, "ttft_relative": 1.240591 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 8, "measured": { "tpot_relative": 1.421552, "ttft_relative": 0.965763 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 93.37, "tpot_relative": 3.299054, "ttft_error_pct": 12.81, "ttft_relative": 1.35443 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 16, "measured": { "tpot_relative": 1.706086, "ttft_relative": 1.200634 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 69.61, "tpot_relative": 3.931658, "ttft_error_pct": 11.75, "ttft_relative": 1.470231 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 32, "measured": { "tpot_relative": 2.318021, "ttft_relative": 1.665951 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 42.53, "tpot_relative": 5.066376, "ttft_error_pct": 42.73, "ttft_relative": 1.587371 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 64, "measured": { "tpot_relative": 3.554617, "ttft_relative": 2.771677 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 28.11, "tpot_relative": 7.392965, "ttft_error_pct": 64.66, "ttft_relative": 1.711494 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 128, "measured": { "tpot_relative": 5.770874, "ttft_relative": 4.842451 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 36.92, "tpot_relative": 12.20266, "ttft_error_pct": 93.86, "ttft_relative": 1.845714 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 256, "measured": { "tpot_relative": 8.912243, "ttft_relative": 30.036205 }, "status": "unsupported" } ], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 9, "tpot_mape_pct": 96.36, "tpot_shape_error_pct": 43.69, "ttft_mape_pct": 35.8, "ttft_shape_error_pct": 41.01 }, "aisimulate": { "points": 0, "status_counts": { "failed": 0, "success": 0, "unknown": 0, "unsupported": 9 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "542111b8521fdfe6", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 221.04, "tpot_relative": 3.210412, "ttft_error_pct": 15.22, "ttft_relative": 0.847752 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 204.5, "tpot_relative": 3.212827, "ttft_error_pct": 11.01, "ttft_relative": 0.964582 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 2, "measured": { "tpot_relative": 1.055099, "ttft_relative": 0.868938 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 121.92, "tpot_relative": 2.598145, "ttft_error_pct": 31.38, "ttft_relative": 1.072267 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 4, "measured": { "tpot_relative": 1.170774, "ttft_relative": 0.816144 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 79.78, "tpot_relative": 2.733775, "ttft_error_pct": 37.07, "ttft_relative": 1.179968 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 8, "measured": { "tpot_relative": 1.520629, "ttft_relative": 1.875071 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 78.02, "tpot_relative": 2.98109, "ttft_error_pct": 10.46, "ttft_relative": 1.288304 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 16, "measured": { "tpot_relative": 1.674609, "ttft_relative": 1.16632 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 58.91, "tpot_relative": 3.525901, "ttft_error_pct": 14.4, "ttft_relative": 1.398616 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 32, "measured": { "tpot_relative": 2.218742, "ttft_relative": 1.633917 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 37.85, "tpot_relative": 4.496301, "ttft_error_pct": 43.64, "ttft_relative": 1.510279 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 64, "measured": { "tpot_relative": 3.26162, "ttft_relative": 2.679842 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 29.38, "tpot_relative": 6.48933, "ttft_error_pct": 65.09, "ttft_relative": 1.628979 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 128, "measured": { "tpot_relative": 5.015813, "ttft_relative": 4.665872 }, "status": "unsupported" }, { "aic": { "tpot_error_pct": 35.81, "tpot_relative": 10.519709, "ttft_error_pct": 93.89, "ttft_relative": 1.757853 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 256, "measured": { "tpot_relative": 7.745973, "ttft_relative": 28.790685 }, "status": "unsupported" } ], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 17, "tpot_mape_pct": 97.81, "tpot_shape_error_pct": 39.3, "ttft_mape_pct": 34.9, "ttft_shape_error_pct": 39.18 }, "aisimulate": { "coverage_pct": 35.29, "points": 6, "status_counts": { "failed": 11, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 111.07, "tpot_shape_error_pct": 25.58, "ttft_mape_pct": 34.14, "ttft_shape_error_pct": 26.76 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 17, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 22.07, "tpot_shape_error_pct": 32.61, "ttft_mape_pct": 45.75, "ttft_shape_error_pct": 63.72 }, "aisimulate": { "points": 0, "status_counts": { "failed": 4, "success": 0, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "dynamo-sglang", "id": "ef9ee52d3d99ba7d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 3.21, "tpot_relative": 0.967851, "ttft_error_pct": 92.35, "ttft_relative": 1.923524 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "failed" }, { "aic": { "tpot_error_pct": 14.87, "tpot_relative": 1.457884, "ttft_error_pct": 15.96, "ttft_relative": 1.923524 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 8, "measured": { "tpot_relative": 1.269152, "ttft_relative": 2.288809 }, "status": "failed" }, { "aic": { "tpot_error_pct": 29.23, "tpot_relative": 1.906584, "ttft_error_pct": 20.76, "ttft_relative": 1.923524 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 16, "measured": { "tpot_relative": 1.475292, "ttft_relative": 2.42744 }, "status": "failed" }, { "aic": { "tpot_error_pct": 40.94, "tpot_relative": 2.443421, "ttft_error_pct": 53.93, "ttft_relative": 1.923524 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 32, "measured": { "tpot_relative": 1.733647, "ttft_relative": 4.175307 }, "status": "failed" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 12.45, "tpot_shape_error_pct": null, "ttft_mape_pct": 63.46, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.55, "tpot_shape_error_pct": null, "ttft_mape_pct": 82.27, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "ad569dc742fa91ce", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 12.45, "tpot_relative": 0.875479, "ttft_error_pct": 63.46, "ttft_relative": 0.365361 }, "aisimulate": { "tpot_error_pct": 12.55, "tpot_relative": 0.874468, "ttft_error_pct": 82.27, "ttft_relative": 0.177267 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 6.8, "tpot_shape_error_pct": null, "ttft_mape_pct": 1.42, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.74, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.7, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "fcdca64ee4be670f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 6.8, "tpot_relative": 1.068025, "ttft_error_pct": 1.42, "ttft_relative": 1.014176 }, "aisimulate": { "tpot_error_pct": 7.74, "tpot_relative": 1.077369, "ttft_error_pct": 31.7, "ttft_relative": 0.682975 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 200.28, "tpot_shape_error_pct": 25.1, "ttft_mape_pct": 21.68, "ttft_shape_error_pct": 42.25 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 199.2, "tpot_shape_error_pct": 25.58, "ttft_mape_pct": 14.13, "ttft_shape_error_pct": 26.76 }, "framework": "vllm", "id": "61002ca4106dbd1c", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 260.62, "tpot_relative": 3.606201, "ttft_error_pct": 10.55, "ttft_relative": 0.894508 }, "aisimulate": { "tpot_error_pct": 260.7, "tpot_relative": 3.60703, "ttft_error_pct": 21.16, "ttft_relative": 0.78843 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 209.0, "tpot_relative": 3.294462, "ttft_error_pct": 10.92, "ttft_relative": 1.016691 }, "aisimulate": { "tpot_error_pct": 206.93, "tpot_relative": 3.272399, "ttft_error_pct": 10.67, "ttft_relative": 0.818766 }, "concurrency": 2, "measured": { "tpot_relative": 1.066176, "ttft_relative": 0.916576 }, "status": "success" }, { "aic": { "tpot_error_pct": 131.22, "tpot_relative": 2.79869, "ttft_error_pct": 43.57, "ttft_relative": 1.129867 }, "aisimulate": { "tpot_error_pct": 129.96, "tpot_relative": 2.783425, "ttft_error_pct": 10.55, "ttft_relative": 0.870031 }, "concurrency": 4, "measured": { "tpot_relative": 1.210409, "ttft_relative": 0.786967 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 130.3, "tpot_shape_error_pct": 47.38, "ttft_mape_pct": 26.68, "ttft_shape_error_pct": 25.88 }, "aisimulate": { "points": 0, "status_counts": { "failed": 7, "success": 0, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "c1bb8d23933c1ea6", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 287.79, "tpot_relative": 3.877855, "ttft_error_pct": 33.79, "ttft_relative": 1.337892 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "failed" }, { "aic": { "tpot_error_pct": 213.17, "tpot_relative": 3.401387, "ttft_error_pct": 40.29, "ttft_relative": 1.52214 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 2, "measured": { "tpot_relative": 1.086107, "ttft_relative": 1.085004 }, "status": "failed" }, { "aic": { "tpot_error_pct": 139.64, "tpot_relative": 2.91698, "ttft_error_pct": 41.09, "ttft_relative": 1.691693 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 4, "measured": { "tpot_relative": 1.217224, "ttft_relative": 1.19902 }, "status": "failed" }, { "aic": { "tpot_error_pct": 101.99, "tpot_relative": 3.008127, "ttft_error_pct": 1.57, "ttft_relative": 1.861514 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 8, "measured": { "tpot_relative": 1.489249, "ttft_relative": 1.891194 }, "status": "failed" }, { "aic": { "tpot_error_pct": 84.73, "tpot_relative": 3.351569, "ttft_error_pct": 6.41, "ttft_relative": 2.03245 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 16, "measured": { "tpot_relative": 1.814344, "ttft_relative": 1.909963 }, "status": "failed" }, { "aic": { "tpot_error_pct": 53.65, "tpot_relative": 3.866395, "ttft_error_pct": 18.96, "ttft_relative": 2.204879 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 32, "measured": { "tpot_relative": 2.516419, "ttft_relative": 2.720641 }, "status": "failed" }, { "aic": { "tpot_error_pct": 31.11, "tpot_relative": 5.034988, "ttft_error_pct": 44.66, "ttft_relative": 2.38005 }, "aisimulate": { "tpot_error_pct": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_relative": null }, "concurrency": 64, "measured": { "tpot_relative": 3.840199, "ttft_relative": 4.30069 }, "status": "failed" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 42.27, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.55, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 48.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 48.51, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "141fe24d39c17dda", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 42.27, "tpot_relative": 1.422707, "ttft_error_pct": 93.55, "ttft_relative": 0.064472 }, "aisimulate": { "tpot_error_pct": 48.56, "tpot_relative": 1.485572, "ttft_error_pct": 48.51, "ttft_relative": 1.485098 }, "concurrency": 2048, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4" ], "rows": 41 } ] }, { "aic": { "points": 67, "tpot_mape_pct": 9.26, "tpot_shape_error_pct": 15.54, "ttft_mape_pct": 29.87, "ttft_shape_error_pct": 32.8 }, "aisimulate": { "coverage_pct": 100.0, "points": 67, "status_counts": { "failed": 0, "success": 67, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.29, "tpot_shape_error_pct": 12.72, "ttft_mape_pct": 44.26, "ttft_shape_error_pct": 16.34 }, "frameworks": [ "sglang" ], "gpu_skus": [ "b200", "b300", "h200" ], "hf_model_paths": [ "nvidia/GLM-5-NVFP4", "zai-org/GLM-5-FP8" ], "model": "GLM-5", "precisions": [ "fp4", "fp8" ], "rows": 67, "workloads": [ { "aic": { "points": 35, "tpot_mape_pct": 8.19, "tpot_shape_error_pct": 13.06, "ttft_mape_pct": 23.58, "ttft_shape_error_pct": 24.98 }, "aisimulate": { "coverage_pct": 100.0, "points": 35, "status_counts": { "failed": 0, "success": 35, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.1, "tpot_shape_error_pct": 13.65, "ttft_mape_pct": 52.38, "ttft_shape_error_pct": 14.71 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 15, "tpot_mape_pct": 7.35, "tpot_shape_error_pct": 13.83, "ttft_mape_pct": 27.78, "ttft_shape_error_pct": 24.08 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.25, "tpot_shape_error_pct": 14.0, "ttft_mape_pct": 49.88, "ttft_shape_error_pct": 9.71 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 15, "topologies": [ { "aic": { "points": 1, "tpot_mape_pct": 16.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.85, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.95, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.69, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "3970e8aec69dd722", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 16.39, "tpot_relative": 0.836128, "ttft_error_pct": 20.85, "ttft_relative": 0.791469 }, "aisimulate": { "tpot_error_pct": 15.95, "tpot_relative": 0.840537, "ttft_error_pct": 62.69, "ttft_relative": 0.373056 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 5.36, "tpot_shape_error_pct": 14.71, "ttft_mape_pct": 19.04, "ttft_shape_error_pct": 22.49 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.83, "tpot_shape_error_pct": 14.93, "ttft_mape_pct": 59.77, "ttft_shape_error_pct": 10.73 }, "framework": "sglang", "id": "08514aec1b1976ac", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.62, "tpot_relative": 0.9138, "ttft_error_pct": 7.91, "ttft_relative": 0.92091 }, "aisimulate": { "tpot_error_pct": 8.27, "tpot_relative": 0.917288, "ttft_error_pct": 57.58, "ttft_relative": 0.424156 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.22, "tpot_relative": 1.182021, "ttft_error_pct": 7.88, "ttft_relative": 1.016678 }, "aisimulate": { "tpot_error_pct": 3.91, "tpot_relative": 1.178486, "ttft_error_pct": 51.59, "ttft_relative": 0.456267 }, "concurrency": 8, "measured": { "tpot_relative": 1.134189, "ttft_relative": 0.942447 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.34, "tpot_relative": 1.418359, "ttft_error_pct": 1.95, "ttft_relative": 1.206285 }, "aisimulate": { "tpot_error_pct": 4.64, "tpot_relative": 1.422397, "ttft_error_pct": 57.82, "ttft_relative": 0.499052 }, "concurrency": 16, "measured": { "tpot_relative": 1.359349, "ttft_relative": 1.183268 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.3, "tpot_relative": 1.730263, "ttft_error_pct": 0.54, "ttft_relative": 1.580991 }, "aisimulate": { "tpot_error_pct": 1.9, "tpot_relative": 1.740649, "ttft_error_pct": 62.75, "ttft_relative": 0.592171 }, "concurrency": 32, "measured": { "tpot_relative": 1.708123, "ttft_relative": 1.589596 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.67, "tpot_relative": 2.307009, "ttft_error_pct": 20.4, "ttft_relative": 1.858098 }, "aisimulate": { "tpot_error_pct": 7.14, "tpot_relative": 2.33907, "ttft_error_pct": 66.8, "ttft_relative": 0.774867 }, "concurrency": 64, "measured": { "tpot_relative": 2.183222, "ttft_relative": 2.334161 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.66, "tpot_relative": 2.886965, "ttft_error_pct": 39.97, "ttft_relative": 1.885123 }, "aisimulate": { "tpot_error_pct": 4.69, "tpot_relative": 2.943985, "ttft_error_pct": 63.83, "ttft_relative": 1.135786 }, "concurrency": 128, "measured": { "tpot_relative": 2.812039, "ttft_relative": 3.140313 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.73, "tpot_relative": 4.160002, "ttft_error_pct": 54.62, "ttft_relative": 2.008001 }, "aisimulate": { "tpot_error_pct": 10.28, "tpot_relative": 4.143052, "ttft_error_pct": 58.03, "ttft_relative": 1.857482 }, "concurrency": 256, "measured": { "tpot_relative": 3.756813, "ttft_relative": 4.425265 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 8.05, "tpot_shape_error_pct": 12.96, "ttft_mape_pct": 37.51, "ttft_shape_error_pct": 25.66 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.42, "tpot_shape_error_pct": 13.06, "ttft_mape_pct": 38.17, "ttft_shape_error_pct": 8.69 }, "framework": "sglang", "id": "84d08c72f86db73e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 17.24, "tpot_relative": 0.8276, "ttft_error_pct": 36.74, "ttft_relative": 1.367361 }, "aisimulate": { "tpot_error_pct": 16.74, "tpot_relative": 0.832556, "ttft_error_pct": 35.27, "ttft_relative": 0.647309 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.85, "tpot_relative": 1.076187, "ttft_error_pct": 46.85, "ttft_relative": 1.507092 }, "aisimulate": { "tpot_error_pct": 6.8, "tpot_relative": 1.076737, "ttft_error_pct": 31.94, "ttft_relative": 0.698456 }, "concurrency": 8, "measured": { "tpot_relative": 1.15535, "ttft_relative": 1.026252 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.46, "tpot_relative": 1.276083, "ttft_error_pct": 57.75, "ttft_relative": 1.778337 }, "aisimulate": { "tpot_error_pct": 6.07, "tpot_relative": 1.281375, "ttft_error_pct": 31.86, "ttft_relative": 0.768144 }, "concurrency": 16, "measured": { "tpot_relative": 1.364225, "ttft_relative": 1.1273 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.9, "tpot_relative": 1.58064, "ttft_error_pct": 62.8, "ttft_relative": 2.338053 }, "aisimulate": { "tpot_error_pct": 7.42, "tpot_relative": 1.58884, "ttft_error_pct": 35.68, "ttft_relative": 0.923662 }, "concurrency": 32, "measured": { "tpot_relative": 1.716238, "ttft_relative": 1.436141 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.96, "tpot_relative": 2.157419, "ttft_error_pct": 7.64, "ttft_relative": 2.755445 }, "aisimulate": { "tpot_error_pct": 0.56, "tpot_relative": 2.188062, "ttft_error_pct": 52.06, "ttft_relative": 1.227191 }, "concurrency": 64, "measured": { "tpot_relative": 2.200469, "ttft_relative": 2.55992 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.71, "tpot_relative": 2.693096, "ttft_error_pct": 14.3, "ttft_relative": 2.804502 }, "aisimulate": { "tpot_error_pct": 8.99, "tpot_relative": 2.745122, "ttft_error_pct": 44.24, "ttft_relative": 1.824496 }, "concurrency": 128, "measured": { "tpot_relative": 3.016163, "ttft_relative": 3.272307 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.21, "tpot_relative": 4.022238, "ttft_error_pct": 36.48, "ttft_relative": 2.998828 }, "aisimulate": { "tpot_error_pct": 5.36, "tpot_relative": 4.015897, "ttft_error_pct": 36.11, "ttft_relative": 3.016426 }, "concurrency": 256, "measured": { "tpot_relative": 4.243532, "ttft_relative": 4.721307 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 15, "tpot_mape_pct": 9.64, "tpot_shape_error_pct": 12.79, "ttft_mape_pct": 24.18, "ttft_shape_error_pct": 27.8 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.73, "tpot_shape_error_pct": 13.82, "ttft_mape_pct": 52.69, "ttft_shape_error_pct": 21.09 }, "gpu": "b300", "precisions": [ "fp4", "fp8" ], "rows": 15, "topologies": [ { "aic": { "points": 7, "tpot_mape_pct": 5.73, "tpot_shape_error_pct": 15.25, "ttft_mape_pct": 16.13, "ttft_shape_error_pct": 18.9 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.5, "tpot_shape_error_pct": 16.74, "ttft_mape_pct": 57.87, "ttft_shape_error_pct": 9.59 }, "framework": "sglang", "id": "e9d9a0c0006e6f05", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.69, "tpot_relative": 0.913076, "ttft_error_pct": 4.85, "ttft_relative": 0.951498 }, "aisimulate": { "tpot_error_pct": 9.15, "tpot_relative": 0.908496, "ttft_error_pct": 55.4, "ttft_relative": 0.445999 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.98, "tpot_relative": 1.148063, "ttft_error_pct": 7.71, "ttft_relative": 1.052129 }, "aisimulate": { "tpot_error_pct": 0.8, "tpot_relative": 1.146024, "ttft_error_pct": 51.29, "ttft_relative": 0.475749 }, "concurrency": 8, "measured": { "tpot_relative": 1.136935, "ttft_relative": 0.976791 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.1, "tpot_relative": 1.430447, "ttft_error_pct": 1.51, "ttft_relative": 1.249049 }, "aisimulate": { "tpot_error_pct": 4.48, "tpot_relative": 1.435691, "ttft_error_pct": 57.69, "ttft_relative": 0.520583 }, "concurrency": 16, "measured": { "tpot_relative": 1.374079, "ttft_relative": 1.230505 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.11, "tpot_relative": 1.703724, "ttft_error_pct": 0.07, "ttft_relative": 1.637712 }, "aisimulate": { "tpot_error_pct": 0.86, "tpot_relative": 1.716522, "ttft_error_pct": 62.2, "ttft_relative": 0.619519 }, "concurrency": 32, "measured": { "tpot_relative": 1.701888, "ttft_relative": 1.638786 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.14, "tpot_relative": 2.301819, "ttft_error_pct": 12.05, "ttft_relative": 1.928704 }, "aisimulate": { "tpot_error_pct": 8.85, "tpot_relative": 2.338499, "ttft_error_pct": 62.92, "ttft_relative": 0.81305 }, "concurrency": 64, "measured": { "tpot_relative": 2.148385, "ttft_relative": 2.192852 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.8, "tpot_relative": 2.835786, "ttft_error_pct": 34.74, "ttft_relative": 1.961351 }, "aisimulate": { "tpot_error_pct": 7.16, "tpot_relative": 2.899629, "ttft_error_pct": 60.26, "ttft_relative": 1.19436 }, "concurrency": 128, "measured": { "tpot_relative": 2.705869, "ttft_relative": 3.005516 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.26, "tpot_relative": 4.087912, "ttft_error_pct": 51.97, "ttft_relative": 2.103444 }, "aisimulate": { "tpot_error_pct": 14.19, "tpot_relative": 4.085185, "ttft_error_pct": 55.31, "ttft_relative": 1.957391 }, "concurrency": 256, "measured": { "tpot_relative": 3.57768, "ttft_relative": 4.37975 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 15.87, "tpot_shape_error_pct": null, "ttft_mape_pct": 6.07, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.79, "tpot_shape_error_pct": null, "ttft_mape_pct": 55.73, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "4510d2cd25dd0226", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 15.87, "tpot_relative": 0.841311, "ttft_error_pct": 6.07, "ttft_relative": 0.939256 }, "aisimulate": { "tpot_error_pct": 15.79, "tpot_relative": 0.842079, "ttft_error_pct": 55.73, "ttft_relative": 0.442671 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 12.66, "tpot_shape_error_pct": 10.33, "ttft_mape_pct": 34.81, "ttft_shape_error_pct": 36.7 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.09, "tpot_shape_error_pct": 10.89, "ttft_mape_pct": 47.08, "ttft_shape_error_pct": 32.59 }, "framework": "sglang", "id": "3aeb3080ad06dbf8", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 19.76, "tpot_relative": 0.802376, "ttft_error_pct": 56.94, "ttft_relative": 1.569397 }, "aisimulate": { "tpot_error_pct": 19.6, "tpot_relative": 0.804029, "ttft_error_pct": 26.56, "ttft_relative": 0.734374 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.97, "tpot_relative": 0.994189, "ttft_error_pct": 37.92, "ttft_relative": 1.72782 }, "aisimulate": { "tpot_error_pct": 13.87, "tpot_relative": 0.995343, "ttft_error_pct": 36.81, "ttft_relative": 0.791583 }, "concurrency": 8, "measured": { "tpot_relative": 1.155634, "ttft_relative": 1.252772 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.95, "tpot_relative": 1.192404, "ttft_error_pct": 27.88, "ttft_relative": 2.035244 }, "aisimulate": { "tpot_error_pct": 12.54, "tpot_relative": 1.198021, "ttft_error_pct": 45.34, "ttft_relative": 0.869868 }, "concurrency": 16, "measured": { "tpot_relative": 1.369867, "ttft_relative": 1.591489 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.06, "tpot_relative": 1.503774, "ttft_error_pct": 25.58, "ttft_relative": 2.670716 }, "aisimulate": { "tpot_error_pct": 9.6, "tpot_relative": 1.511548, "ttft_error_pct": 50.91, "ttft_relative": 1.043929 }, "concurrency": 32, "measured": { "tpot_relative": 1.672035, "ttft_relative": 2.126721 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.67, "tpot_relative": 2.07594, "ttft_error_pct": 9.49, "ttft_relative": 3.147877 }, "aisimulate": { "tpot_error_pct": 5.49, "tpot_relative": 2.102189, "ttft_error_pct": 60.21, "ttft_relative": 1.383876 }, "concurrency": 64, "measured": { "tpot_relative": 2.224402, "ttft_relative": 3.477991 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.14, "tpot_relative": 2.612016, "ttft_error_pct": 33.22, "ttft_relative": 3.191593 }, "aisimulate": { "tpot_error_pct": 11.4, "tpot_relative": 2.664562, "ttft_error_pct": 57.04, "ttft_relative": 2.05302 }, "concurrency": 128, "measured": { "tpot_relative": 3.007297, "ttft_relative": 4.779075 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.05, "tpot_relative": 3.801605, "ttft_error_pct": 52.66, "ttft_relative": 3.388717 }, "aisimulate": { "tpot_error_pct": 12.13, "tpot_relative": 3.798097, "ttft_error_pct": 52.66, "ttft_relative": 3.388585 }, "concurrency": 256, "measured": { "tpot_relative": 4.322464, "ttft_relative": 7.158676 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 5, "tpot_mape_pct": 6.38, "tpot_shape_error_pct": 11.54, "ttft_mape_pct": 9.18, "ttft_shape_error_pct": 19.23 }, "aisimulate": { "coverage_pct": 100.0, "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.76, "tpot_shape_error_pct": 12.1, "ttft_mape_pct": 58.93, "ttft_shape_error_pct": 10.59 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 5, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 6.38, "tpot_shape_error_pct": 11.54, "ttft_mape_pct": 9.18, "ttft_shape_error_pct": 19.23 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.76, "tpot_shape_error_pct": 12.1, "ttft_mape_pct": 58.93, "ttft_shape_error_pct": 10.59 }, "framework": "sglang", "id": "15b9dacd0eb879a0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 11.54, "tpot_relative": 0.884582, "ttft_error_pct": 21.29, "ttft_relative": 0.787148 }, "aisimulate": { "tpot_error_pct": 11.15, "tpot_relative": 0.888482, "ttft_error_pct": 62.13, "ttft_relative": 0.37866 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.52, "tpot_relative": 1.154369, "ttft_error_pct": 12.4, "ttft_relative": 0.867948 }, "aisimulate": { "tpot_error_pct": 3.3, "tpot_relative": 1.156995, "ttft_error_pct": 58.38, "ttft_relative": 0.412388 }, "concurrency": 8, "measured": { "tpot_relative": 1.196537, "ttft_relative": 0.990795 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.52, "tpot_relative": 1.638676, "ttft_error_pct": 8.92, "ttft_relative": 1.032935 }, "aisimulate": { "tpot_error_pct": 8.02, "tpot_relative": 1.646217, "ttft_error_pct": 59.67, "ttft_relative": 0.457373 }, "concurrency": 16, "measured": { "tpot_relative": 1.524026, "ttft_relative": 1.134047 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.38, "tpot_relative": 1.98609, "ttft_error_pct": 1.4, "ttft_relative": 1.367282 }, "aisimulate": { "tpot_error_pct": 4.38, "tpot_relative": 2.007018, "ttft_error_pct": 59.78, "ttft_relative": 0.557702 }, "concurrency": 32, "measured": { "tpot_relative": 2.098999, "ttft_relative": 1.386645 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.95, "tpot_relative": 2.700657, "ttft_error_pct": 1.88, "ttft_relative": 1.62974 }, "aisimulate": { "tpot_error_pct": 1.95, "tpot_relative": 2.757034, "ttft_error_pct": 54.67, "ttft_relative": 0.752935 }, "concurrency": 64, "measured": { "tpot_relative": 2.811858, "ttft_relative": 1.660954 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4", "fp8" ], "rows": 35 }, { "aic": { "points": 32, "tpot_mape_pct": 10.42, "tpot_shape_error_pct": 18.31, "ttft_mape_pct": 36.76, "ttft_shape_error_pct": 41.55 }, "aisimulate": { "coverage_pct": 100.0, "points": 32, "status_counts": { "failed": 0, "success": 32, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.51, "tpot_shape_error_pct": 11.68, "ttft_mape_pct": 35.38, "ttft_shape_error_pct": 18.16 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 12, "tpot_mape_pct": 5.5, "tpot_shape_error_pct": 15.68, "ttft_mape_pct": 23.45, "ttft_shape_error_pct": 36.8 }, "aisimulate": { "coverage_pct": 100.0, "points": 12, "status_counts": { "failed": 0, "success": 12, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.75, "tpot_shape_error_pct": 11.17, "ttft_mape_pct": 45.6, "ttft_shape_error_pct": 35.79 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 12, "topologies": [ { "aic": { "points": 1, "tpot_mape_pct": 17.38, "tpot_shape_error_pct": null, "ttft_mape_pct": 26.99, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.17, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.05, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "2e5c3c732f7edb88", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 17.38, "tpot_relative": 0.826195, "ttft_error_pct": 26.99, "ttft_relative": 1.269934 }, "aisimulate": { "tpot_error_pct": 13.17, "tpot_relative": 0.868252, "ttft_error_pct": 36.05, "ttft_relative": 0.639513 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 2.9, "tpot_shape_error_pct": 11.69, "ttft_mape_pct": 19.82, "ttft_shape_error_pct": 30.13 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 2.98, "tpot_shape_error_pct": 8.34, "ttft_mape_pct": 41.04, "ttft_shape_error_pct": 9.29 }, "framework": "sglang", "id": "6ba52ae27882ad03", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 9.43, "tpot_relative": 0.905724, "ttft_error_pct": 27.44, "ttft_relative": 1.274436 }, "aisimulate": { "tpot_error_pct": 5.55, "tpot_relative": 0.944534, "ttft_error_pct": 36.31, "ttft_relative": 0.636866 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.15, "tpot_relative": 1.215302, "ttft_error_pct": 13.9, "ttft_relative": 1.401266 }, "aisimulate": { "tpot_error_pct": 3.93, "tpot_relative": 1.248634, "ttft_error_pct": 38.42, "ttft_relative": 0.757581 }, "concurrency": 8, "measured": { "tpot_relative": 1.201434, "ttft_relative": 1.230231 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.22, "tpot_relative": 1.578901, "ttft_error_pct": 5.23, "ttft_relative": 1.653807 }, "aisimulate": { "tpot_error_pct": 1.64, "tpot_relative": 1.60834, "ttft_error_pct": 44.44, "ttft_relative": 0.969613 }, "concurrency": 16, "measured": { "tpot_relative": 1.582374, "ttft_relative": 1.745033 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.16, "tpot_relative": 2.246265, "ttft_error_pct": 14.67, "ttft_relative": 2.165962 }, "aisimulate": { "tpot_error_pct": 2.2, "tpot_relative": 2.246963, "ttft_error_pct": 43.52, "ttft_relative": 1.433746 }, "concurrency": 32, "measured": { "tpot_relative": 2.198686, "ttft_relative": 2.538391 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.53, "tpot_relative": 3.388648, "ttft_error_pct": 37.84, "ttft_relative": 2.523408 }, "aisimulate": { "tpot_error_pct": 1.57, "tpot_relative": 3.39009, "ttft_error_pct": 42.54, "ttft_relative": 2.332644 }, "concurrency": 64, "measured": { "tpot_relative": 3.337673, "ttft_relative": 4.059308 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 5.68, "tpot_shape_error_pct": 18.88, "ttft_mape_pct": 25.88, "ttft_shape_error_pct": 42.13 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.83, "tpot_shape_error_pct": 13.43, "ttft_mape_pct": 50.98, "ttft_shape_error_pct": 57.0 }, "framework": "sglang", "id": "4b1292ec78bd34ac", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 18.2, "tpot_relative": 0.817985, "ttft_error_pct": 61.93, "ttft_relative": 1.619333 }, "aisimulate": { "tpot_error_pct": 14.07, "tpot_relative": 0.85926, "ttft_error_pct": 18.16, "ttft_relative": 0.818387 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.12, "tpot_relative": 1.1159, "ttft_error_pct": 12.06, "ttft_relative": 1.779536 }, "aisimulate": { "tpot_error_pct": 5.25, "tpot_relative": 1.15081, "ttft_error_pct": 38.67, "ttft_relative": 0.973956 }, "concurrency": 8, "measured": { "tpot_relative": 1.214523, "ttft_relative": 1.587972 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.56, "tpot_relative": 1.473569, "ttft_error_pct": 9.59, "ttft_relative": 2.099583 }, "aisimulate": { "tpot_error_pct": 3.53, "tpot_relative": 1.505275, "ttft_error_pct": 34.95, "ttft_relative": 1.246319 }, "concurrency": 16, "measured": { "tpot_relative": 1.560351, "ttft_relative": 1.915928 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.16, "tpot_relative": 2.116378, "ttft_error_pct": 9.29, "ttft_relative": 2.750702 }, "aisimulate": { "tpot_error_pct": 0.77, "tpot_relative": 2.12457, "ttft_error_pct": 26.77, "ttft_relative": 1.843101 }, "concurrency": 32, "measured": { "tpot_relative": 2.141117, "ttft_relative": 2.516772 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.74, "tpot_relative": 3.249838, "ttft_error_pct": 15.11, "ttft_relative": 3.206086 }, "aisimulate": { "tpot_error_pct": 1.11, "tpot_relative": 3.261863, "ttft_error_pct": 20.6, "ttft_relative": 2.998721 }, "concurrency": 64, "measured": { "tpot_relative": 3.225993, "ttft_relative": 3.776901 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.3, "tpot_relative": 5.250098, "ttft_error_pct": 47.29, "ttft_relative": 3.267609 }, "aisimulate": { "tpot_error_pct": 4.23, "tpot_relative": 5.01278, "ttft_error_pct": 166.74, "ttft_relative": 16.536701 }, "concurrency": 128, "measured": { "tpot_relative": 5.234424, "ttft_relative": 6.199591 }, "status": "success" } ], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 15, "tpot_mape_pct": 11.12, "tpot_shape_error_pct": 12.41, "ttft_mape_pct": 33.0, "ttft_shape_error_pct": 46.36 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.97, "tpot_shape_error_pct": 8.34, "ttft_mape_pct": 37.72, "ttft_shape_error_pct": 9.35 }, "gpu": "b300", "precisions": [ "fp4", "fp8" ], "rows": 15, "topologies": [ { "aic": { "points": 7, "tpot_mape_pct": 4.8, "tpot_shape_error_pct": 15.25, "ttft_mape_pct": 33.62, "ttft_shape_error_pct": 46.94 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.11, "tpot_shape_error_pct": 11.17, "ttft_mape_pct": 36.47, "ttft_shape_error_pct": 8.5 }, "framework": "sglang", "id": "97e112e44b6d338e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 10.44, "tpot_relative": 0.895552, "ttft_error_pct": 36.99, "ttft_relative": 1.369926 }, "aisimulate": { "tpot_error_pct": 6.75, "tpot_relative": 0.932535, "ttft_error_pct": 31.48, "ttft_relative": 0.685188 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.95, "tpot_relative": 1.173886, "ttft_error_pct": 17.23, "ttft_relative": 1.506377 }, "aisimulate": { "tpot_error_pct": 0.79, "tpot_relative": 1.206625, "ttft_error_pct": 36.56, "ttft_relative": 0.815123 }, "concurrency": 8, "measured": { "tpot_relative": 1.19719, "ttft_relative": 1.284973 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.17, "tpot_relative": 1.581307, "ttft_error_pct": 1.35, "ttft_relative": 1.778547 }, "aisimulate": { "tpot_error_pct": 3.06, "tpot_relative": 1.610793, "ttft_error_pct": 42.13, "ttft_relative": 1.043229 }, "concurrency": 16, "measured": { "tpot_relative": 1.562949, "ttft_relative": 1.802825 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.9, "tpot_relative": 2.203376, "ttft_error_pct": 9.4, "ttft_relative": 2.330047 }, "aisimulate": { "tpot_error_pct": 2.83, "tpot_relative": 2.20193, "ttft_error_pct": 40.02, "ttft_relative": 1.542628 }, "concurrency": 32, "measured": { "tpot_relative": 2.141263, "ttft_relative": 2.571837 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.87, "tpot_relative": 3.334299, "ttft_error_pct": 34.84, "ttft_relative": 2.714048 }, "aisimulate": { "tpot_error_pct": 4.03, "tpot_relative": 3.339584, "ttft_error_pct": 39.75, "ttft_relative": 2.5098 }, "concurrency": 64, "measured": { "tpot_relative": 3.210081, "ttft_relative": 4.1655 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.81, "tpot_relative": 5.317145, "ttft_error_pct": 58.73, "ttft_relative": 2.767325 }, "aisimulate": { "tpot_error_pct": 6.23, "tpot_relative": 5.288127, "ttft_error_pct": 33.85, "ttft_relative": 4.435522 }, "concurrency": 128, "measured": { "tpot_relative": 4.978213, "ttft_relative": 6.705302 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.46, "tpot_relative": 9.080357, "ttft_error_pct": 76.8, "ttft_relative": 2.861462 }, "aisimulate": { "tpot_error_pct": 5.1, "tpot_relative": 8.964212, "ttft_error_pct": 31.51, "ttft_relative": 8.44638 }, "concurrency": 256, "measured": { "tpot_relative": 8.529452, "ttft_relative": 12.332202 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 17.83, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.88, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.83, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.53, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "a69f64c588ac6de2", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 17.83, "tpot_relative": 0.821657, "ttft_error_pct": 35.88, "ttft_relative": 1.358777 }, "aisimulate": { "tpot_error_pct": 13.83, "tpot_relative": 0.861676, "ttft_error_pct": 31.53, "ttft_relative": 0.684724 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 16.47, "tpot_shape_error_pct": 9.58, "ttft_mape_pct": 31.97, "ttft_shape_error_pct": 45.77 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.28, "tpot_shape_error_pct": 5.5, "ttft_mape_pct": 39.85, "ttft_shape_error_pct": 10.2 }, "framework": "sglang", "id": "c6faa867cfa39895", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 22.81, "tpot_relative": 0.771904, "ttft_error_pct": 22.54, "ttft_relative": 1.225417 }, "aisimulate": { "tpot_error_pct": 19.1, "tpot_relative": 0.809031, "ttft_error_pct": 38.1, "ttft_relative": 0.618961 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.17, "tpot_relative": 1.012062, "ttft_error_pct": 8.51, "ttft_relative": 1.346114 }, "aisimulate": { "tpot_error_pct": 15.53, "tpot_relative": 1.044721, "ttft_error_pct": 49.93, "ttft_relative": 0.736605 }, "concurrency": 8, "measured": { "tpot_relative": 1.236805, "ttft_relative": 1.471245 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.12, "tpot_relative": 1.359392, "ttft_error_pct": 11.54, "ttft_relative": 1.58826 }, "aisimulate": { "tpot_error_pct": 14.41, "tpot_relative": 1.387018, "ttft_error_pct": 47.5, "ttft_relative": 0.942602 }, "concurrency": 16, "measured": { "tpot_relative": 1.620592, "ttft_relative": 1.795542 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.69, "tpot_relative": 1.983145, "ttft_error_pct": 13.31, "ttft_relative": 2.080801 }, "aisimulate": { "tpot_error_pct": 12.18, "tpot_relative": 1.994864, "ttft_error_pct": 41.93, "ttft_relative": 1.393947 }, "concurrency": 32, "measured": { "tpot_relative": 2.271431, "ttft_relative": 2.40032 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.41, "tpot_relative": 3.053272, "ttft_error_pct": 32.75, "ttft_relative": 2.427316 }, "aisimulate": { "tpot_error_pct": 13.08, "tpot_relative": 3.064946, "ttft_error_pct": 37.16, "ttft_relative": 2.267946 }, "concurrency": 64, "measured": { "tpot_relative": 3.526284, "ttft_relative": 3.609328 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.06, "tpot_relative": 4.931429, "ttft_error_pct": 58.52, "ttft_relative": 2.474857 }, "aisimulate": { "tpot_error_pct": 15.25, "tpot_relative": 4.920485, "ttft_error_pct": 32.81, "ttft_relative": 4.008534 }, "concurrency": 128, "measured": { "tpot_relative": 5.805653, "ttft_relative": 5.965703 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.04, "tpot_relative": 8.475906, "ttft_error_pct": 76.66, "ttft_relative": 2.554894 }, "aisimulate": { "tpot_error_pct": 17.42, "tpot_relative": 8.436739, "ttft_error_pct": 31.51, "ttft_relative": 7.495695 }, "concurrency": 256, "measured": { "tpot_relative": 10.216695, "ttft_relative": 10.944859 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 5, "tpot_mape_pct": 20.15, "tpot_shape_error_pct": 41.92, "ttft_mape_pct": 79.98, "ttft_shape_error_pct": 37.81 }, "aisimulate": { "coverage_pct": 100.0, "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.12, "tpot_shape_error_pct": 22.85, "ttft_mape_pct": 3.85, "ttft_shape_error_pct": 4.93 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 5, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 20.15, "tpot_shape_error_pct": 41.92, "ttft_mape_pct": 79.98, "ttft_shape_error_pct": 37.81 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.12, "tpot_shape_error_pct": 22.85, "ttft_mape_pct": 3.85, "ttft_shape_error_pct": 4.93 }, "framework": "sglang", "id": "81411dec84b826ec", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 14.32, "tpot_relative": 0.85685, "ttft_error_pct": 105.72, "ttft_relative": 2.05724 }, "aisimulate": { "tpot_error_pct": 6.59, "tpot_relative": 0.934066, "ttft_error_pct": 4.19, "ttft_relative": 1.041913 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.88, "tpot_relative": 1.257604, "ttft_error_pct": 87.71, "ttft_relative": 2.259507 }, "aisimulate": { "tpot_error_pct": 6.12, "tpot_relative": 1.322896, "ttft_error_pct": 2.98, "ttft_relative": 1.239617 }, "concurrency": 8, "measured": { "tpot_relative": 1.246619, "ttft_relative": 1.203744 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.38, "tpot_relative": 2.000929, "ttft_error_pct": 62.78, "ttft_relative": 2.665397 }, "aisimulate": { "tpot_error_pct": 17.55, "tpot_relative": 2.056489, "ttft_error_pct": 3.11, "ttft_relative": 1.586453 }, "concurrency": 16, "measured": { "tpot_relative": 1.749403, "ttft_relative": 1.637454 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.93, "tpot_relative": 3.001005, "ttft_error_pct": 52.46, "ttft_relative": 3.483681 }, "aisimulate": { "tpot_error_pct": 13.9, "tpot_relative": 3.026873, "ttft_error_pct": 2.67, "ttft_relative": 2.345948 }, "concurrency": 32, "measured": { "tpot_relative": 2.657494, "ttft_relative": 2.284974 }, "status": "success" }, { "aic": { "tpot_error_pct": 58.23, "tpot_relative": 4.919631, "ttft_error_pct": 91.22, "ttft_relative": 4.059393 }, "aisimulate": { "tpot_error_pct": 21.45, "tpot_relative": 3.775901, "ttft_error_pct": 6.3, "ttft_relative": 43.327112 }, "concurrency": 64, "measured": { "tpot_relative": 3.109112, "ttft_relative": 46.2414 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4", "fp8" ], "rows": 32 } ] }, { "aic": { "points": 150, "tpot_mape_pct": 43.17, "tpot_shape_error_pct": 22.71, "ttft_mape_pct": 67.9, "ttft_shape_error_pct": 32.75 }, "aisimulate": { "coverage_pct": 100.0, "points": 150, "status_counts": { "failed": 0, "success": 150, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 44.97, "tpot_shape_error_pct": 24.45, "ttft_mape_pct": 59.59, "ttft_shape_error_pct": 16.66 }, "frameworks": [ "trtllm", "vllm" ], "gpu_skus": [ "b200", "h100", "h200" ], "hf_model_paths": [ "openai/gpt-oss-120b" ], "model": "gpt-oss-120b", "precisions": [ "fp4" ], "rows": 150, "workloads": [ { "aic": { "points": 70, "tpot_mape_pct": 42.33, "tpot_shape_error_pct": 31.72, "ttft_mape_pct": 60.62, "ttft_shape_error_pct": 30.35 }, "aisimulate": { "coverage_pct": 100.0, "points": 70, "status_counts": { "failed": 0, "success": 70, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 45.47, "tpot_shape_error_pct": 35.77, "ttft_mape_pct": 54.42, "ttft_shape_error_pct": 17.84 }, "gpu_skus": [ "b200", "h100", "h200" ], "gpus": [ { "aic": { "points": 27, "tpot_mape_pct": 56.57, "tpot_shape_error_pct": 17.64, "ttft_mape_pct": 101.93, "ttft_shape_error_pct": 25.79 }, "aisimulate": { "coverage_pct": 100.0, "points": 27, "status_counts": { "failed": 0, "success": 27, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 60.48, "tpot_shape_error_pct": 20.49, "ttft_mape_pct": 50.2, "ttft_shape_error_pct": 8.61 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 27, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 198.8, "tpot_shape_error_pct": 48.07, "ttft_mape_pct": 278.38, "ttft_shape_error_pct": 18.3 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 215.09, "tpot_shape_error_pct": 55.27, "ttft_mape_pct": 151.42, "ttft_shape_error_pct": 14.47 }, "framework": "trtllm", "id": "7023bea79c28e9fc", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 115.8, "tpot_relative": 2.158041, "ttft_error_pct": 327.16, "ttft_relative": 4.271576 }, "aisimulate": { "tpot_error_pct": 118.48, "tpot_relative": 2.184843, "ttft_error_pct": 131.54, "ttft_relative": 2.315425 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 164.22, "tpot_relative": 3.402054, "ttft_error_pct": 348.44, "ttft_relative": 4.688097 }, "aisimulate": { "tpot_error_pct": 165.95, "tpot_relative": 3.424353, "ttft_error_pct": 155.89, "ttft_relative": 2.675199 }, "concurrency": 8, "measured": { "tpot_relative": 1.287605, "ttft_relative": 1.04543 }, "status": "success" }, { "aic": { "tpot_error_pct": 188.81, "tpot_relative": 4.762682, "ttft_error_pct": 340.3, "ttft_relative": 5.520693 }, "aisimulate": { "tpot_error_pct": 192.06, "tpot_relative": 4.816301, "ttft_error_pct": 148.49, "ttft_relative": 3.115746 }, "concurrency": 16, "measured": { "tpot_relative": 1.649059, "ttft_relative": 1.25385 }, "status": "success" }, { "aic": { "tpot_error_pct": 189.53, "tpot_relative": 6.441421, "ttft_error_pct": 296.41, "ttft_relative": 7.199518 }, "aisimulate": { "tpot_error_pct": 194.46, "tpot_relative": 6.551263, "ttft_error_pct": 114.22, "ttft_relative": 3.890546 }, "concurrency": 32, "measured": { "tpot_relative": 2.224818, "ttft_relative": 1.816187 }, "status": "success" }, { "aic": { "tpot_error_pct": 335.64, "tpot_relative": 19.702209, "ttft_error_pct": 79.6, "ttft_relative": 8.529625 }, "aisimulate": { "tpot_error_pct": 404.51, "tpot_relative": 22.817196, "ttft_error_pct": 206.95, "ttft_relative": 14.578003 }, "concurrency": 256, "measured": { "tpot_relative": 4.522631, "ttft_relative": 4.749272 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 17.39, "tpot_shape_error_pct": 5.8, "ttft_mape_pct": 59.98, "ttft_shape_error_pct": 24.43 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.26, "tpot_shape_error_pct": 7.72, "ttft_mape_pct": 39.19, "ttft_shape_error_pct": 7.38 }, "framework": "vllm", "id": "dac5fc374991bd1e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 12.18, "tpot_relative": 1.121846, "ttft_error_pct": 98.84, "ttft_relative": 1.98845 }, "aisimulate": { "tpot_error_pct": 11.38, "tpot_relative": 1.113769, "ttft_error_pct": 35.37, "ttft_relative": 0.646305 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.01, "tpot_relative": 1.294107, "ttft_error_pct": 76.85, "ttft_relative": 2.187522 }, "aisimulate": { "tpot_error_pct": 16.63, "tpot_relative": 1.289872, "ttft_error_pct": 40.24, "ttft_relative": 0.739198 }, "concurrency": 8, "measured": { "tpot_relative": 1.105999, "ttft_relative": 1.23696 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.77, "tpot_relative": 1.481422, "ttft_error_pct": 66.29, "ttft_relative": 2.388054 }, "aisimulate": { "tpot_error_pct": 18.26, "tpot_relative": 1.487545, "ttft_error_pct": 40.74, "ttft_relative": 0.851033 }, "concurrency": 16, "measured": { "tpot_relative": 1.257902, "ttft_relative": 1.436045 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.61, "tpot_relative": 1.778323, "ttft_error_pct": 41.87, "ttft_relative": 2.589803 }, "aisimulate": { "tpot_error_pct": 15.96, "tpot_relative": 1.79922, "ttft_error_pct": 41.53, "ttft_relative": 1.06731 }, "concurrency": 32, "measured": { "tpot_relative": 1.551619, "ttft_relative": 1.825445 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.38, "tpot_relative": 2.3768, "ttft_error_pct": 16.04, "ttft_relative": 2.794226 }, "aisimulate": { "tpot_error_pct": 29.07, "tpot_relative": 2.446683, "ttft_error_pct": 38.05, "ttft_relative": 1.491594 }, "concurrency": 64, "measured": { "tpot_relative": 1.895698, "ttft_relative": 2.407932 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 34.81, "tpot_shape_error_pct": 15.06, "ttft_mape_pct": 71.76, "ttft_shape_error_pct": 35.01 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 36.17, "tpot_shape_error_pct": 18.13, "ttft_mape_pct": 12.76, "ttft_shape_error_pct": 6.87 }, "framework": "vllm", "id": "6ea9e584d4aba1f1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 19.78, "tpot_relative": 1.197832, "ttft_error_pct": 138.45, "ttft_relative": 2.384514 }, "aisimulate": { "tpot_error_pct": 18.3, "tpot_relative": 1.182978, "ttft_error_pct": 19.61, "ttft_relative": 1.196107 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 33.75, "tpot_relative": 1.62673, "ttft_error_pct": 116.86, "ttft_relative": 2.6234 }, "aisimulate": { "tpot_error_pct": 32.49, "tpot_relative": 1.611426, "ttft_error_pct": 16.07, "ttft_relative": 1.404162 }, "concurrency": 8, "measured": { "tpot_relative": 1.216268, "ttft_relative": 1.209743 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.05, "tpot_relative": 2.076978, "ttft_error_pct": 92.8, "ttft_relative": 2.862877 }, "aisimulate": { "tpot_error_pct": 27.72, "tpot_relative": 2.071594, "ttft_error_pct": 10.66, "ttft_relative": 1.643215 }, "concurrency": 16, "measured": { "tpot_relative": 1.622026, "ttft_relative": 1.484885 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.82, "tpot_relative": 2.799949, "ttft_error_pct": 48.16, "ttft_relative": 3.108666 }, "aisimulate": { "tpot_error_pct": 31.25, "tpot_relative": 2.809092, "ttft_error_pct": 0.12, "ttft_relative": 2.100624 }, "concurrency": 32, "measured": { "tpot_relative": 2.140254, "ttft_relative": 2.098134 }, "status": "success" }, { "aic": { "tpot_error_pct": 46.45, "tpot_relative": 4.190305, "ttft_error_pct": 25.65, "ttft_relative": 3.364667 }, "aisimulate": { "tpot_error_pct": 49.77, "tpot_relative": 4.285075, "ttft_error_pct": 12.04, "ttft_relative": 3.000141 }, "concurrency": 64, "measured": { "tpot_relative": 2.861162, "ttft_relative": 2.677748 }, "status": "success" }, { "aic": { "tpot_error_pct": 50.02, "tpot_relative": 5.500854, "ttft_error_pct": 8.62, "ttft_relative": 3.63309 }, "aisimulate": { "tpot_error_pct": 57.49, "tpot_relative": 5.774703, "ttft_error_pct": 18.06, "ttft_relative": 4.693858 }, "concurrency": 128, "measured": { "tpot_relative": 3.666801, "ttft_relative": 3.97567 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 8.14, "tpot_shape_error_pct": 8.89, "ttft_mape_pct": 48.7, "ttft_shape_error_pct": 25.44 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.4, "tpot_shape_error_pct": 8.13, "ttft_mape_pct": 50.16, "ttft_shape_error_pct": 10.0 }, "framework": "vllm", "id": "559262f676b611bd", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 7.52, "tpot_relative": 1.075179, "ttft_error_pct": 86.7, "ttft_relative": 1.866956 }, "aisimulate": { "tpot_error_pct": 7.16, "tpot_relative": 1.071603, "ttft_error_pct": 47.0, "ttft_relative": 0.530021 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.55, "tpot_relative": 1.244705, "ttft_error_pct": 71.49, "ttft_relative": 2.054213 }, "aisimulate": { "tpot_error_pct": 13.28, "tpot_relative": 1.241757, "ttft_error_pct": 49.3, "ttft_relative": 0.607297 }, "concurrency": 8, "measured": { "tpot_relative": 1.0962, "ttft_relative": 1.197849 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.21, "tpot_relative": 1.440098, "ttft_error_pct": 65.27, "ttft_relative": 2.242964 }, "aisimulate": { "tpot_error_pct": 10.76, "tpot_relative": 1.44731, "ttft_error_pct": 48.32, "ttft_relative": 0.701322 }, "concurrency": 16, "measured": { "tpot_relative": 1.62187, "ttft_relative": 1.357175 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.43, "tpot_relative": 1.858954, "ttft_error_pct": 3.81, "ttft_relative": 2.431519 }, "aisimulate": { "tpot_error_pct": 2.1, "tpot_relative": 1.884697, "ttft_error_pct": 61.87, "ttft_relative": 0.893008 }, "concurrency": 32, "measured": { "tpot_relative": 1.925033, "ttft_relative": 2.342238 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.01, "tpot_relative": 2.516008, "ttft_error_pct": 16.26, "ttft_relative": 2.623588 }, "aisimulate": { "tpot_error_pct": 8.71, "tpot_relative": 2.604691, "ttft_error_pct": 44.29, "ttft_relative": 1.257139 }, "concurrency": 64, "measured": { "tpot_relative": 2.396068, "ttft_relative": 2.256698 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 32.8, "tpot_shape_error_pct": 12.36, "ttft_mape_pct": 64.36, "ttft_shape_error_pct": 23.94 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 34.51, "tpot_shape_error_pct": 15.12, "ttft_mape_pct": 12.5, "ttft_shape_error_pct": 5.54 }, "framework": "vllm", "id": "3ccb300d01c6d04d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 20.4, "tpot_relative": 1.203987, "ttft_error_pct": 103.04, "ttft_relative": 2.03039 }, "aisimulate": { "tpot_error_pct": 19.46, "tpot_relative": 1.194603, "ttft_error_pct": 16.36, "ttft_relative": 0.836368 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 26.36, "tpot_relative": 1.454955, "ttft_error_pct": 102.46, "ttft_relative": 2.233837 }, "aisimulate": { "tpot_error_pct": 25.79, "tpot_relative": 1.448458, "ttft_error_pct": 12.73, "ttft_relative": 0.962854 }, "concurrency": 8, "measured": { "tpot_relative": 1.15146, "ttft_relative": 1.103363 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.39, "tpot_relative": 1.771839, "ttft_error_pct": 84.4, "ttft_relative": 2.438314 }, "aisimulate": { "tpot_error_pct": 27.48, "tpot_relative": 1.773134, "ttft_error_pct": 15.56, "ttft_relative": 1.116584 }, "concurrency": 16, "measured": { "tpot_relative": 1.390905, "ttft_relative": 1.32231 }, "status": "success" }, { "aic": { "tpot_error_pct": 29.59, "tpot_relative": 2.212813, "ttft_error_pct": 57.62, "ttft_relative": 2.644065 }, "aisimulate": { "tpot_error_pct": 30.68, "tpot_relative": 2.231392, "ttft_error_pct": 16.1, "ttft_relative": 1.407377 }, "concurrency": 32, "measured": { "tpot_relative": 1.707514, "ttft_relative": 1.677472 }, "status": "success" }, { "aic": { "tpot_error_pct": 43.37, "tpot_relative": 3.058436, "ttft_error_pct": 33.19, "ttft_relative": 2.861033 }, "aisimulate": { "tpot_error_pct": 46.56, "tpot_relative": 3.126463, "ttft_error_pct": 8.12, "ttft_relative": 1.973622 }, "concurrency": 64, "measured": { "tpot_relative": 2.133216, "ttft_relative": 2.148159 }, "status": "success" }, { "aic": { "tpot_error_pct": 49.68, "tpot_relative": 4.100674, "ttft_error_pct": 5.46, "ttft_relative": 3.0941 }, "aisimulate": { "tpot_error_pct": 57.09, "tpot_relative": 4.303901, "ttft_error_pct": 6.15, "ttft_relative": 3.071508 }, "concurrency": 128, "measured": { "tpot_relative": 2.73971, "ttft_relative": 3.272671 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 18, "tpot_mape_pct": 72.06, "tpot_shape_error_pct": 89.02, "ttft_mape_pct": 33.43, "ttft_shape_error_pct": 41.78 }, "aisimulate": { "coverage_pct": 100.0, "points": 18, "status_counts": { "failed": 0, "success": 18, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 79.22, "tpot_shape_error_pct": 100.05, "ttft_mape_pct": 59.42, "ttft_shape_error_pct": 29.35 }, "gpu": "h100", "precisions": [ "fp4" ], "rows": 18, "topologies": [ { "aic": { "points": 6, "tpot_mape_pct": 104.45, "tpot_shape_error_pct": 126.94, "ttft_mape_pct": 28.91, "ttft_shape_error_pct": 39.71 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 116.95, "tpot_shape_error_pct": 143.88, "ttft_mape_pct": 55.21, "ttft_shape_error_pct": 27.47 }, "framework": "vllm", "id": "381949ac4a70b539", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 0.96, "tpot_relative": 0.990397, "ttft_error_pct": 12.44, "ttft_relative": 1.124392 }, "aisimulate": { "tpot_error_pct": 1.92, "tpot_relative": 0.980757, "ttft_error_pct": 42.56, "ttft_relative": 0.574426 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.64, "tpot_relative": 1.278597, "ttft_error_pct": 1.55, "ttft_relative": 1.238287 }, "aisimulate": { "tpot_error_pct": 4.05, "tpot_relative": 1.271325, "ttft_error_pct": 47.82, "ttft_relative": 0.656295 }, "concurrency": 8, "measured": { "tpot_relative": 1.221871, "ttft_relative": 1.257796 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.08, "tpot_relative": 1.618728, "ttft_error_pct": 3.69, "ttft_relative": 1.35392 }, "aisimulate": { "tpot_error_pct": 2.18, "tpot_relative": 1.620376, "ttft_error_pct": 45.92, "ttft_relative": 0.760295 }, "concurrency": 16, "measured": { "tpot_relative": 1.585781, "ttft_relative": 1.405745 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.68, "tpot_relative": 2.011277, "ttft_error_pct": 9.11, "ttft_relative": 1.47446 }, "aisimulate": { "tpot_error_pct": 5.03, "tpot_relative": 2.0251, "ttft_error_pct": 41.07, "ttft_relative": 0.956 }, "concurrency": 32, "measured": { "tpot_relative": 2.132347, "ttft_relative": 1.62216 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.44, "tpot_relative": 2.957434, "ttft_error_pct": 47.33, "ttft_relative": 1.609403 }, "aisimulate": { "tpot_error_pct": 5.0, "tpot_relative": 3.031303, "ttft_error_pct": 55.85, "ttft_relative": 1.34894 }, "concurrency": 64, "measured": { "tpot_relative": 2.887008, "ttft_relative": 3.055556 }, "status": "success" }, { "aic": { "tpot_error_pct": 610.9, "tpot_relative": 7.760874, "ttft_error_pct": 99.36, "ttft_relative": 2.16147 }, "aisimulate": { "tpot_error_pct": 683.51, "tpot_relative": 8.553533, "ttft_error_pct": 98.05, "ttft_relative": 6.608537 }, "concurrency": 512, "measured": { "tpot_relative": 1.091695, "ttft_relative": 338.10793 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 71.71, "tpot_shape_error_pct": 92.27, "ttft_mape_pct": 32.95, "ttft_shape_error_pct": 41.59 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 77.93, "tpot_shape_error_pct": 102.89, "ttft_mape_pct": 59.65, "ttft_shape_error_pct": 28.58 }, "framework": "vllm", "id": "8a27b634beb4d68b", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 7.54, "tpot_relative": 0.924603, "ttft_error_pct": 27.01, "ttft_relative": 1.270066 }, "aisimulate": { "tpot_error_pct": 8.1, "tpot_relative": 0.918985, "ttft_error_pct": 47.03, "ttft_relative": 0.529665 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.47, "tpot_relative": 1.140163, "ttft_error_pct": 17.39, "ttft_relative": 1.398181 }, "aisimulate": { "tpot_error_pct": 2.9, "tpot_relative": 1.135147, "ttft_error_pct": 49.23, "ttft_relative": 0.604751 }, "concurrency": 8, "measured": { "tpot_relative": 1.169032, "ttft_relative": 1.191074 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.31, "tpot_relative": 1.368823, "ttft_error_pct": 3.42, "ttft_relative": 1.527876 }, "aisimulate": { "tpot_error_pct": 4.06, "tpot_relative": 1.372455, "ttft_error_pct": 52.71, "ttft_relative": 0.69856 }, "concurrency": 16, "measured": { "tpot_relative": 1.430462, "ttft_relative": 1.477281 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.46, "tpot_relative": 1.711228, "ttft_error_pct": 16.39, "ttft_relative": 1.66068 }, "aisimulate": { "tpot_error_pct": 4.36, "tpot_relative": 1.731076, "ttft_error_pct": 55.5, "ttft_relative": 0.884001 }, "concurrency": 32, "measured": { "tpot_relative": 1.809989, "ttft_relative": 1.986312 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.7, "tpot_relative": 2.268274, "ttft_error_pct": 34.1, "ttft_relative": 1.803199 }, "aisimulate": { "tpot_error_pct": 2.3, "tpot_relative": 2.325526, "ttft_error_pct": 54.93, "ttft_relative": 1.233244 }, "concurrency": 64, "measured": { "tpot_relative": 2.380235, "ttft_relative": 2.736096 }, "status": "success" }, { "aic": { "tpot_error_pct": 405.79, "tpot_relative": 5.799509, "ttft_error_pct": 99.42, "ttft_relative": 2.330715 }, "aisimulate": { "tpot_error_pct": 445.89, "tpot_relative": 6.259226, "ttft_error_pct": 98.49, "ttft_relative": 6.043411 }, "concurrency": 512, "measured": { "tpot_relative": 1.146616, "ttft_relative": 399.835147 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 40.02, "tpot_shape_error_pct": 47.85, "ttft_mape_pct": 38.42, "ttft_shape_error_pct": 44.06 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 42.78, "tpot_shape_error_pct": 53.39, "ttft_mape_pct": 63.4, "ttft_shape_error_pct": 31.99 }, "framework": "vllm", "id": "090acf5801eee832", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 8.88, "tpot_relative": 0.911152, "ttft_error_pct": 41.91, "ttft_relative": 1.419061 }, "aisimulate": { "tpot_error_pct": 9.19, "tpot_relative": 0.908143, "ttft_error_pct": 50.1, "ttft_relative": 0.49903 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.91, "tpot_relative": 0.997357, "ttft_error_pct": 28.73, "ttft_relative": 1.561978 }, "aisimulate": { "tpot_error_pct": 7.99, "tpot_relative": 0.996484, "ttft_error_pct": 53.57, "ttft_relative": 0.563376 }, "concurrency": 8, "measured": { "tpot_relative": 1.082973, "ttft_relative": 1.213398 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.2, "tpot_relative": 1.211472, "ttft_error_pct": 14.05, "ttft_relative": 1.706178 }, "aisimulate": { "tpot_error_pct": 5.84, "tpot_relative": 1.216111, "ttft_error_pct": 56.2, "ttft_relative": 0.65525 }, "concurrency": 16, "measured": { "tpot_relative": 1.291595, "ttft_relative": 1.495963 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.07, "tpot_relative": 1.397961, "ttft_error_pct": 23.03, "ttft_relative": 1.853494 }, "aisimulate": { "tpot_error_pct": 8.81, "tpot_relative": 1.417459, "ttft_error_pct": 65.78, "ttft_relative": 0.824114 }, "concurrency": 32, "measured": { "tpot_relative": 1.554482, "ttft_relative": 2.407934 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.84, "tpot_relative": 1.853461, "ttft_error_pct": 23.25, "ttft_relative": 2.00703 }, "aisimulate": { "tpot_error_pct": 2.87, "tpot_relative": 1.911926, "ttft_error_pct": 55.75, "ttft_relative": 1.15728 }, "concurrency": 64, "measured": { "tpot_relative": 1.968363, "ttft_relative": 2.615158 }, "status": "success" }, { "aic": { "tpot_error_pct": 201.24, "tpot_relative": 4.2842, "ttft_error_pct": 99.56, "ttft_relative": 2.515135 }, "aisimulate": { "tpot_error_pct": 222.01, "tpot_relative": 4.579673, "ttft_error_pct": 99.01, "ttft_relative": 5.625735 }, "concurrency": 512, "measured": { "tpot_relative": 1.422206, "ttft_relative": 569.867606 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 25, "tpot_mape_pct": 5.55, "tpot_shape_error_pct": 4.22, "ttft_mape_pct": 35.6, "ttft_shape_error_pct": 26.8 }, "aisimulate": { "coverage_pct": 100.0, "points": 25, "status_counts": { "failed": 0, "success": 25, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.96, "tpot_shape_error_pct": 4.36, "ttft_mape_pct": 55.39, "ttft_shape_error_pct": 19.35 }, "gpu": "h200", "precisions": [ "fp4" ], "rows": 25, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 6.48, "tpot_shape_error_pct": 6.94, "ttft_mape_pct": 71.16, "ttft_shape_error_pct": 18.75 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.17, "tpot_shape_error_pct": 7.77, "ttft_mape_pct": 80.51, "ttft_shape_error_pct": 32.39 }, "framework": "trtllm", "id": "192d544bb3feb0af", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 11.4, "tpot_relative": 0.885987, "ttft_error_pct": 74.92, "ttft_relative": 0.250765 }, "aisimulate": { "tpot_error_pct": 10.72, "tpot_relative": 0.892836, "ttft_error_pct": 84.52, "ttft_relative": 0.154751 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.46, "tpot_relative": 1.282658, "ttft_error_pct": 64.74, "ttft_relative": 0.277281 }, "aisimulate": { "tpot_error_pct": 3.15, "tpot_relative": 1.28678, "ttft_error_pct": 76.78, "ttft_relative": 0.182616 }, "concurrency": 8, "measured": { "tpot_relative": 1.328603, "ttft_relative": 0.786495 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.49, "tpot_relative": 1.720494, "ttft_error_pct": 69.2, "ttft_relative": 0.330676 }, "aisimulate": { "tpot_error_pct": 1.63, "tpot_relative": 1.735565, "ttft_error_pct": 79.63, "ttft_relative": 0.218733 }, "concurrency": 16, "measured": { "tpot_relative": 1.764411, "ttft_relative": 1.073557 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.23, "tpot_relative": 2.100323, "ttft_error_pct": 72.22, "ttft_relative": 0.435985 }, "aisimulate": { "tpot_error_pct": 8.66, "tpot_relative": 2.137096, "ttft_error_pct": 81.9, "ttft_relative": 0.284008 }, "concurrency": 32, "measured": { "tpot_relative": 2.339745, "ttft_relative": 1.569198 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.83, "tpot_relative": 3.044461, "ttft_error_pct": 74.73, "ttft_relative": 0.516892 }, "aisimulate": { "tpot_error_pct": 1.68, "tpot_relative": 3.145461, "ttft_error_pct": 79.74, "ttft_relative": 0.41431 }, "concurrency": 64, "measured": { "tpot_relative": 3.199138, "ttft_relative": 2.045118 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 8.63, "tpot_shape_error_pct": 1.93, "ttft_mape_pct": 27.63, "ttft_shape_error_pct": 20.66 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.83, "tpot_shape_error_pct": 1.8, "ttft_mape_pct": 53.51, "ttft_shape_error_pct": 9.53 }, "framework": "vllm", "id": "74d9f693f9ee9705", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 8.7, "tpot_relative": 0.912982, "ttft_error_pct": 36.88, "ttft_relative": 1.368764 }, "aisimulate": { "tpot_error_pct": 9.03, "tpot_relative": 0.909653, "ttft_error_pct": 52.95, "ttft_relative": 0.470504 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.25, "tpot_relative": 0.989719, "ttft_error_pct": 30.96, "ttft_relative": 1.506454 }, "aisimulate": { "tpot_error_pct": 9.31, "tpot_relative": 0.989116, "ttft_error_pct": 54.15, "ttft_relative": 0.527406 }, "concurrency": 8, "measured": { "tpot_relative": 1.090657, "ttft_relative": 1.150308 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.0, "tpot_relative": 1.210222, "ttft_error_pct": 46.78, "ttft_relative": 1.645907 }, "aisimulate": { "tpot_error_pct": 4.66, "tpot_relative": 1.214596, "ttft_error_pct": 45.39, "ttft_relative": 0.612321 }, "concurrency": 16, "measured": { "tpot_relative": 1.273928, "ttft_relative": 1.121361 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.63, "tpot_relative": 1.375478, "ttft_error_pct": 2.34, "ttft_relative": 1.787672 }, "aisimulate": { "tpot_error_pct": 8.36, "tpot_relative": 1.394746, "ttft_error_pct": 58.21, "ttft_relative": 0.764998 }, "concurrency": 32, "measured": { "tpot_relative": 1.521986, "ttft_relative": 1.830526 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.57, "tpot_relative": 1.714398, "ttft_error_pct": 21.2, "ttft_relative": 1.935355 }, "aisimulate": { "tpot_error_pct": 7.8, "tpot_relative": 1.767379, "ttft_error_pct": 56.87, "ttft_relative": 1.059328 }, "concurrency": 64, "measured": { "tpot_relative": 1.916968, "ttft_relative": 2.45605 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 2.48, "tpot_shape_error_pct": 2.51, "ttft_mape_pct": 20.26, "ttft_shape_error_pct": 25.76 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 2.64, "tpot_shape_error_pct": 3.37, "ttft_mape_pct": 45.32, "ttft_shape_error_pct": 11.98 }, "framework": "vllm", "id": "57f7b1a10332db68", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 0.76, "tpot_relative": 1.007611, "ttft_error_pct": 18.32, "ttft_relative": 1.183153 }, "aisimulate": { "tpot_error_pct": 0.25, "tpot_relative": 0.997452, "ttft_error_pct": 40.11, "ttft_relative": 0.598857 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.48, "tpot_relative": 1.275647, "ttft_error_pct": 17.15, "ttft_relative": 1.302916 }, "aisimulate": { "tpot_error_pct": 2.89, "tpot_relative": 1.268272, "ttft_error_pct": 38.79, "ttft_relative": 0.68082 }, "concurrency": 8, "measured": { "tpot_relative": 1.232704, "ttft_relative": 1.112201 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.97, "tpot_relative": 1.646029, "ttft_error_pct": 1.61, "ttft_relative": 1.424474 }, "aisimulate": { "tpot_error_pct": 1.02, "tpot_relative": 1.646944, "ttft_error_pct": 45.44, "ttft_relative": 0.78994 }, "concurrency": 16, "measured": { "tpot_relative": 1.630285, "ttft_relative": 1.447742 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.98, "tpot_relative": 2.05199, "ttft_error_pct": 21.82, "ttft_relative": 1.549618 }, "aisimulate": { "tpot_error_pct": 5.21, "tpot_relative": 2.068975, "ttft_error_pct": 49.95, "ttft_relative": 0.99206 }, "concurrency": 32, "measured": { "tpot_relative": 2.182609, "ttft_relative": 1.982159 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.19, "tpot_relative": 3.016998, "ttft_error_pct": 42.39, "ttft_relative": 1.687923 }, "aisimulate": { "tpot_error_pct": 3.82, "tpot_relative": 3.095603, "ttft_error_pct": 52.32, "ttft_relative": 1.396864 }, "concurrency": 64, "measured": { "tpot_relative": 2.981583, "ttft_relative": 2.929792 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 3.53, "tpot_shape_error_pct": 4.99, "ttft_mape_pct": 18.5, "ttft_shape_error_pct": 23.3 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 2.99, "tpot_shape_error_pct": 3.42, "ttft_mape_pct": 37.87, "ttft_shape_error_pct": 9.58 }, "framework": "vllm", "id": "d10957b26935f01f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 0.83, "tpot_relative": 1.008265, "ttft_error_pct": 10.1, "ttft_relative": 1.100992 }, "aisimulate": { "tpot_error_pct": 0.41, "tpot_relative": 0.995907, "ttft_error_pct": 33.79, "ttft_relative": 0.66211 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.31, "tpot_relative": 1.424003, "ttft_error_pct": 10.35, "ttft_relative": 1.212457 }, "aisimulate": { "tpot_error_pct": 1.18, "tpot_relative": 1.411588, "ttft_error_pct": 31.3, "ttft_relative": 0.754833 }, "concurrency": 8, "measured": { "tpot_relative": 1.42849, "ttft_relative": 1.098779 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.68, "tpot_relative": 1.921446, "ttft_error_pct": 4.13, "ttft_relative": 1.325762 }, "aisimulate": { "tpot_error_pct": 3.85, "tpot_relative": 1.918104, "ttft_error_pct": 36.7, "ttft_relative": 0.875426 }, "concurrency": 16, "measured": { "tpot_relative": 1.994893, "ttft_relative": 1.382911 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.12, "tpot_relative": 2.499663, "ttft_error_pct": 23.73, "ttft_relative": 1.445821 }, "aisimulate": { "tpot_error_pct": 9.45, "tpot_relative": 2.518301, "ttft_error_pct": 42.08, "ttft_relative": 1.097834 }, "concurrency": 32, "measured": { "tpot_relative": 2.781175, "ttft_relative": 1.895567 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.71, "tpot_relative": 3.798609, "ttft_error_pct": 44.2, "ttft_relative": 1.579403 }, "aisimulate": { "tpot_error_pct": 0.05, "tpot_relative": 3.902554, "ttft_error_pct": 45.46, "ttft_relative": 1.543574 }, "concurrency": 64, "measured": { "tpot_relative": 3.904433, "ttft_relative": 2.830352 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 6.65, "tpot_shape_error_pct": 4.74, "ttft_mape_pct": 40.43, "ttft_shape_error_pct": 45.51 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.19, "tpot_shape_error_pct": 5.43, "ttft_mape_pct": 59.73, "ttft_shape_error_pct": 33.28 }, "framework": "vllm", "id": "0cfdf9389d1fc3a0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.03, "tpot_relative": 0.919672, "ttft_error_pct": 33.05, "ttft_relative": 1.330453 }, "aisimulate": { "tpot_error_pct": 8.62, "tpot_relative": 0.913835, "ttft_error_pct": 45.18, "ttft_relative": 0.548178 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.56, "tpot_relative": 1.115306, "ttft_error_pct": 29.56, "ttft_relative": 1.464677 }, "aisimulate": { "tpot_error_pct": 3.99, "tpot_relative": 1.110312, "ttft_error_pct": 45.07, "ttft_relative": 0.621015 }, "concurrency": 8, "measured": { "tpot_relative": 1.156458, "ttft_relative": 1.130521 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.45, "tpot_relative": 1.308252, "ttft_error_pct": 98.36, "ttft_relative": 1.600215 }, "aisimulate": { "tpot_error_pct": 1.22, "tpot_relative": 1.311316, "ttft_error_pct": 99.27, "ttft_relative": 0.712245 }, "concurrency": 16, "measured": { "tpot_relative": 1.327484, "ttft_relative": 97.581016 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.3, "tpot_relative": 1.561545, "ttft_error_pct": 6.8, "ttft_relative": 1.738499 }, "aisimulate": { "tpot_error_pct": 12.49, "tpot_relative": 1.576005, "ttft_error_pct": 52.38, "ttft_relative": 0.8883 }, "concurrency": 32, "measured": { "tpot_relative": 1.800991, "ttft_relative": 1.865294 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.93, "tpot_relative": 2.200256, "ttft_error_pct": 34.41, "ttft_relative": 1.883168 }, "aisimulate": { "tpot_error_pct": 4.65, "tpot_relative": 2.254168, "ttft_error_pct": 56.75, "ttft_relative": 1.241715 }, "concurrency": 64, "measured": { "tpot_relative": 2.364119, "ttft_relative": 2.871264 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4" ], "rows": 70 }, { "aic": { "points": 11, "tpot_mape_pct": 67.46, "tpot_shape_error_pct": 16.09, "ttft_mape_pct": 141.83, "ttft_shape_error_pct": 16.88 }, "aisimulate": { "coverage_pct": 100.0, "points": 11, "status_counts": { "failed": 0, "success": 11, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 66.54, "tpot_shape_error_pct": 16.04, "ttft_mape_pct": 79.88, "ttft_shape_error_pct": 7.07 }, "gpu_skus": [ "b200", "h200" ], "gpus": [ { "aic": { "points": 7, "tpot_mape_pct": 104.33, "tpot_shape_error_pct": 23.91, "ttft_mape_pct": 213.02, "ttft_shape_error_pct": 11.73 }, "aisimulate": { "coverage_pct": 100.0, "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 103.24, "tpot_shape_error_pct": 23.73, "ttft_mape_pct": 103.65, "ttft_shape_error_pct": 6.53 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 7, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 177.49, "tpot_shape_error_pct": 28.16, "ttft_mape_pct": 343.91, "ttft_shape_error_pct": 7.11 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 175.69, "tpot_shape_error_pct": 27.86, "ttft_mape_pct": 145.3, "ttft_shape_error_pct": 6.0 }, "framework": "trtllm", "id": "173735b71c6f9c5e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 129.1, "tpot_relative": 2.290988, "ttft_error_pct": 368.92, "ttft_relative": 4.689238 }, "aisimulate": { "tpot_error_pct": 128.05, "tpot_relative": 2.280489, "ttft_error_pct": 154.12, "ttft_relative": 2.541221 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 168.76, "tpot_relative": 3.578003, "ttft_error_pct": 355.78, "ttft_relative": 5.150837 }, "aisimulate": { "tpot_error_pct": 167.13, "tpot_relative": 3.556202, "ttft_error_pct": 159.37, "ttft_relative": 2.93118 }, "concurrency": 8, "measured": { "tpot_relative": 1.331279, "ttft_relative": 1.130105 }, "status": "success" }, { "aic": { "tpot_error_pct": 212.1, "tpot_relative": 4.990178, "ttft_error_pct": 328.61, "ttft_relative": 6.085989 }, "aisimulate": { "tpot_error_pct": 209.59, "tpot_relative": 4.950028, "ttft_error_pct": 141.38, "ttft_relative": 3.427459 }, "concurrency": 16, "measured": { "tpot_relative": 1.598915, "ttft_relative": 1.419949 }, "status": "success" }, { "aic": { "tpot_error_pct": 199.99, "tpot_relative": 6.634775, "ttft_error_pct": 322.32, "ttft_relative": 7.961877 }, "aisimulate": { "tpot_error_pct": 198.01, "tpot_relative": 6.5909, "ttft_error_pct": 126.34, "ttft_relative": 4.267206 }, "concurrency": 32, "measured": { "tpot_relative": 2.21164, "ttft_relative": 1.885269 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 6.79, "tpot_shape_error_pct": 17.54, "ttft_mape_pct": 38.51, "ttft_shape_error_pct": 18.64 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.63, "tpot_shape_error_pct": 17.55, "ttft_mape_pct": 48.1, "ttft_shape_error_pct": 7.33 }, "framework": "vllm", "id": "ac16954048a9e9bf", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 10.9, "tpot_relative": 0.891012, "ttft_error_pct": 58.16, "ttft_relative": 1.581643 }, "aisimulate": { "tpot_error_pct": 11.25, "tpot_relative": 0.887479, "ttft_error_pct": 50.52, "ttft_relative": 0.494811 }, "concurrency": 16, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.68, "tpot_relative": 1.133805, "ttft_error_pct": 44.22, "ttft_relative": 1.714946 }, "aisimulate": { "tpot_error_pct": 2.22, "tpot_relative": 1.128716, "ttft_error_pct": 47.4, "ttft_relative": 0.62542 }, "concurrency": 32, "measured": { "tpot_relative": 1.104241, "ttft_relative": 1.189094 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.79, "tpot_relative": 1.534431, "ttft_error_pct": 13.13, "ttft_relative": 1.853776 }, "aisimulate": { "tpot_error_pct": 6.43, "tpot_relative": 1.529319, "ttft_error_pct": 46.38, "ttft_relative": 0.878561 }, "concurrency": 64, "measured": { "tpot_relative": 1.4369, "ttft_relative": 1.638607 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 4, "tpot_mape_pct": 2.92, "tpot_shape_error_pct": 3.06, "ttft_mape_pct": 17.26, "ttft_shape_error_pct": 25.48 }, "aisimulate": { "coverage_pct": 100.0, "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 2.31, "tpot_shape_error_pct": 3.21, "ttft_mape_pct": 38.3, "ttft_shape_error_pct": 7.95 }, "gpu": "h200", "precisions": [ "fp4" ], "rows": 4, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 2.92, "tpot_shape_error_pct": 3.06, "ttft_mape_pct": 17.26, "ttft_shape_error_pct": 25.48 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 2.31, "tpot_shape_error_pct": 3.21, "ttft_mape_pct": 38.3, "ttft_shape_error_pct": 7.95 }, "framework": "vllm", "id": "356a55e61d7ba5f8", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 3.74, "tpot_relative": 1.037432, "ttft_error_pct": 6.0, "ttft_relative": 1.059972 }, "aisimulate": { "tpot_error_pct": 2.61, "tpot_relative": 1.026067, "ttft_error_pct": 34.38, "ttft_relative": 0.656154 }, "concurrency": 8, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.05, "tpot_relative": 1.394077, "ttft_error_pct": 3.73, "ttft_relative": 1.162309 }, "aisimulate": { "tpot_error_pct": 1.9, "tpot_relative": 1.378544, "ttft_error_pct": 36.49, "ttft_relative": 0.766817 }, "concurrency": 16, "measured": { "tpot_relative": 1.352858, "ttft_relative": 1.207332 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.12, "tpot_relative": 1.804005, "ttft_error_pct": 19.01, "ttft_relative": 1.27393 }, "aisimulate": { "tpot_error_pct": 4.34, "tpot_relative": 1.78121, "ttft_error_pct": 39.3, "ttft_relative": 0.954858 }, "concurrency": 32, "measured": { "tpot_relative": 1.862111, "ttft_relative": 1.573043 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.78, "tpot_relative": 2.750357, "ttft_error_pct": 40.28, "ttft_relative": 1.404112 }, "aisimulate": { "tpot_error_pct": 0.4, "tpot_relative": 2.713023, "ttft_error_pct": 43.03, "ttft_relative": 1.339532 }, "concurrency": 64, "measured": { "tpot_relative": 2.702238, "ttft_relative": 2.351186 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:8192", "label": "1k8k", "precisions": [ "fp4" ], "rows": 11 }, { "aic": { "points": 69, "tpot_mape_pct": 40.15, "tpot_shape_error_pct": 14.48, "ttft_mape_pct": 63.5, "ttft_shape_error_pct": 37.45 }, "aisimulate": { "coverage_pct": 100.0, "points": 69, "status_counts": { "failed": 0, "success": 69, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 41.03, "tpot_shape_error_pct": 14.12, "ttft_mape_pct": 61.6, "ttft_shape_error_pct": 16.83 }, "gpu_skus": [ "b200", "h100", "h200" ], "gpus": [ { "aic": { "points": 29, "tpot_mape_pct": 80.14, "tpot_shape_error_pct": 22.94, "ttft_mape_pct": 80.52, "ttft_shape_error_pct": 37.61 }, "aisimulate": { "coverage_pct": 100.0, "points": 29, "status_counts": { "failed": 0, "success": 29, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 81.62, "tpot_shape_error_pct": 23.48, "ttft_mape_pct": 68.9, "ttft_shape_error_pct": 12.81 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 29, "topologies": [ { "aic": { "points": 7, "tpot_mape_pct": 235.46, "tpot_shape_error_pct": 46.28, "ttft_mape_pct": 239.8, "ttft_shape_error_pct": 35.8 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 247.63, "tpot_shape_error_pct": 44.42, "ttft_mape_pct": 206.26, "ttft_shape_error_pct": 20.07 }, "framework": "trtllm", "id": "617dbb03979ec581", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 140.18, "tpot_relative": 2.401752, "ttft_error_pct": 390.26, "ttft_relative": 4.902593 }, "aisimulate": { "tpot_error_pct": 151.77, "tpot_relative": 2.517717, "ttft_error_pct": 161.3, "ttft_relative": 2.613045 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 208.87, "tpot_relative": 4.016091, "ttft_error_pct": 367.1, "ttft_relative": 5.387364 }, "aisimulate": { "tpot_error_pct": 216.51, "tpot_relative": 4.115492, "ttft_error_pct": 182.27, "ttft_relative": 3.255555 }, "concurrency": 8, "measured": { "tpot_relative": 1.300272, "ttft_relative": 1.153368 }, "status": "success" }, { "aic": { "tpot_error_pct": 230.01, "tpot_relative": 5.980796, "ttft_error_pct": 331.13, "ttft_relative": 6.35892 }, "aisimulate": { "tpot_error_pct": 235.81, "tpot_relative": 6.085777, "ttft_error_pct": 180.77, "ttft_relative": 4.141232 }, "concurrency": 16, "measured": { "tpot_relative": 1.812285, "ttft_relative": 1.474938 }, "status": "success" }, { "aic": { "tpot_error_pct": 243.91, "tpot_relative": 8.765374, "ttft_error_pct": 310.31, "ttft_relative": 8.324181 }, "aisimulate": { "tpot_error_pct": 249.09, "tpot_relative": 8.897483, "ttft_error_pct": 200.53, "ttft_relative": 6.097086 }, "concurrency": 32, "measured": { "tpot_relative": 2.548728, "ttft_relative": 2.028758 }, "status": "success" }, { "aic": { "tpot_error_pct": 282.03, "tpot_relative": 14.951953, "ttft_error_pct": 196.08, "ttft_relative": 9.733471 }, "aisimulate": { "tpot_error_pct": 290.58, "tpot_relative": 15.286399, "ttft_error_pct": 208.06, "ttft_relative": 10.127516 }, "concurrency": 64, "measured": { "tpot_relative": 3.913795, "ttft_relative": 3.287494 }, "status": "success" }, { "aic": { "tpot_error_pct": 259.92, "tpot_relative": 22.720322, "ttft_error_pct": 82.63, "ttft_relative": 9.909622 }, "aisimulate": { "tpot_error_pct": 274.23, "tpot_relative": 23.623883, "ttft_error_pct": 244.16, "ttft_relative": 18.674724 }, "concurrency": 128, "measured": { "tpot_relative": 6.312615, "ttft_relative": 5.426098 }, "status": "success" }, { "aic": { "tpot_error_pct": 283.29, "tpot_relative": 37.327918, "ttft_error_pct": 1.1, "ttft_relative": 10.257783 }, "aisimulate": { "tpot_error_pct": 315.39, "tpot_relative": 40.453694, "ttft_error_pct": 266.71, "ttft_relative": 37.205939 }, "concurrency": 256, "measured": { "tpot_relative": 9.738815, "ttft_relative": 10.145782 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 22.42, "tpot_shape_error_pct": 10.73, "ttft_mape_pct": 22.87, "ttft_shape_error_pct": 36.09 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 20.76, "tpot_shape_error_pct": 11.87, "ttft_mape_pct": 28.59, "ttft_shape_error_pct": 5.38 }, "framework": "vllm", "id": "6997f9bd9329bb9f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 12.74, "tpot_relative": 1.127438, "ttft_error_pct": 22.74, "ttft_relative": 1.227414 }, "aisimulate": { "tpot_error_pct": 10.29, "tpot_relative": 1.102852, "ttft_error_pct": 28.09, "ttft_relative": 0.719116 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.11, "tpot_relative": 1.36272, "ttft_error_pct": 2.69, "ttft_relative": 1.351203 }, "aisimulate": { "tpot_error_pct": 19.83, "tpot_relative": 1.33729, "ttft_error_pct": 32.61, "ttft_relative": 0.886786 }, "concurrency": 8, "measured": { "tpot_relative": 1.115991, "ttft_relative": 1.315847 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.86, "tpot_relative": 1.65736, "ttft_error_pct": 7.83, "ttft_relative": 1.475293 }, "aisimulate": { "tpot_error_pct": 18.99, "tpot_relative": 1.631774, "ttft_error_pct": 30.1, "ttft_relative": 1.118911 }, "concurrency": 16, "measured": { "tpot_relative": 1.371297, "ttft_relative": 1.600692 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.55, "tpot_relative": 2.266729, "ttft_error_pct": 31.91, "ttft_relative": 1.604731 }, "aisimulate": { "tpot_error_pct": 23.02, "tpot_relative": 2.238966, "ttft_error_pct": 30.56, "ttft_relative": 1.636525 }, "concurrency": 32, "measured": { "tpot_relative": 1.819938, "ttft_relative": 2.356811 }, "status": "success" }, { "aic": { "tpot_error_pct": 31.84, "tpot_relative": 3.419441, "ttft_error_pct": 49.18, "ttft_relative": 1.738698 }, "aisimulate": { "tpot_error_pct": 31.67, "tpot_relative": 3.414979, "ttft_error_pct": 21.6, "ttft_relative": 2.681998 }, "concurrency": 64, "measured": { "tpot_relative": 2.593564, "ttft_relative": 3.421128 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 48.07, "tpot_shape_error_pct": 21.61, "ttft_mape_pct": 40.01, "ttft_shape_error_pct": 45.05 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 44.47, "tpot_shape_error_pct": 24.22, "ttft_mape_pct": 26.62, "ttft_shape_error_pct": 6.94 }, "framework": "vllm", "id": "cb2d830228801835", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 25.47, "tpot_relative": 1.254724, "ttft_error_pct": 74.88, "ttft_relative": 1.748796 }, "aisimulate": { "tpot_error_pct": 20.21, "tpot_relative": 1.202077, "ttft_error_pct": 26.14, "ttft_relative": 1.261396 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 44.81, "tpot_relative": 1.836823, "ttft_error_pct": 46.3, "ttft_relative": 1.92653 }, "aisimulate": { "tpot_error_pct": 39.6, "tpot_relative": 1.77067, "ttft_error_pct": 18.65, "ttft_relative": 1.562503 }, "concurrency": 8, "measured": { "tpot_relative": 1.268433, "ttft_relative": 1.316849 }, "status": "success" }, { "aic": { "tpot_error_pct": 44.17, "tpot_relative": 2.575275, "ttft_error_pct": 26.55, "ttft_relative": 2.107357 }, "aisimulate": { "tpot_error_pct": 40.15, "tpot_relative": 2.503454, "ttft_error_pct": 19.18, "ttft_relative": 1.984708 }, "concurrency": 16, "measured": { "tpot_relative": 1.786263, "ttft_relative": 1.665241 }, "status": "success" }, { "aic": { "tpot_error_pct": 46.79, "tpot_relative": 3.880025, "ttft_error_pct": 5.37, "ttft_relative": 2.295702 }, "aisimulate": { "tpot_error_pct": 43.41, "tpot_relative": 3.79059, "ttft_error_pct": 20.15, "ttft_relative": 2.914659 }, "concurrency": 32, "measured": { "tpot_relative": 2.643217, "ttft_relative": 2.425926 }, "status": "success" }, { "aic": { "tpot_error_pct": 61.01, "tpot_relative": 6.424786, "ttft_error_pct": 32.33, "ttft_relative": 2.502721 }, "aisimulate": { "tpot_error_pct": 58.74, "tpot_relative": 6.334303, "ttft_error_pct": 29.91, "ttft_relative": 4.804974 }, "concurrency": 64, "measured": { "tpot_relative": 3.990374, "ttft_relative": 3.698619 }, "status": "success" }, { "aic": { "tpot_error_pct": 66.15, "tpot_relative": 10.264315, "ttft_error_pct": 54.63, "ttft_relative": 2.737884 }, "aisimulate": { "tpot_error_pct": 64.71, "tpot_relative": 10.175263, "ttft_error_pct": 45.7, "ttft_relative": 8.792481 }, "concurrency": 128, "measured": { "tpot_relative": 6.177773, "ttft_relative": 6.034816 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 7.19, "tpot_shape_error_pct": 10.57, "ttft_mape_pct": 21.67, "ttft_shape_error_pct": 25.0 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.56, "tpot_shape_error_pct": 10.24, "ttft_mape_pct": 39.35, "ttft_shape_error_pct": 21.15 }, "framework": "vllm", "id": "82406f57cd28f490", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 7.25, "tpot_relative": 1.072509, "ttft_error_pct": 4.16, "ttft_relative": 0.958409 }, "aisimulate": { "tpot_error_pct": 5.36, "tpot_relative": 1.053575, "ttft_error_pct": 48.13, "ttft_relative": 0.518708 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.65, "tpot_relative": 1.290289, "ttft_error_pct": 1.99, "ttft_relative": 1.054956 }, "aisimulate": { "tpot_error_pct": 11.13, "tpot_relative": 1.269183, "ttft_error_pct": 38.08, "ttft_relative": 0.640438 }, "concurrency": 8, "measured": { "tpot_relative": 1.428126, "ttft_relative": 1.034336 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.25, "tpot_relative": 1.572016, "ttft_error_pct": 7.88, "ttft_relative": 1.151866 }, "aisimulate": { "tpot_error_pct": 10.19, "tpot_relative": 1.555627, "ttft_error_pct": 35.35, "ttft_relative": 0.808375 }, "concurrency": 16, "measured": { "tpot_relative": 1.73218, "ttft_relative": 1.250408 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.62, "tpot_relative": 2.176344, "ttft_error_pct": 44.1, "ttft_relative": 1.249634 }, "aisimulate": { "tpot_error_pct": 4.11, "tpot_relative": 2.165259, "ttft_error_pct": 46.98, "ttft_relative": 1.185211 }, "concurrency": 32, "measured": { "tpot_relative": 2.258007, "ttft_relative": 2.235557 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.17, "tpot_relative": 3.296609, "ttft_error_pct": 50.19, "ttft_relative": 1.353728 }, "aisimulate": { "tpot_error_pct": 6.99, "tpot_relative": 3.322001, "ttft_error_pct": 28.21, "ttft_relative": 1.951225 }, "concurrency": 64, "measured": { "tpot_relative": 3.104903, "ttft_relative": 2.718053 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 39.89, "tpot_shape_error_pct": 15.93, "ttft_mape_pct": 32.27, "ttft_shape_error_pct": 43.64 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 37.52, "tpot_shape_error_pct": 17.51, "ttft_mape_pct": 9.14, "ttft_shape_error_pct": 9.25 }, "framework": "vllm", "id": "cb724c6af21e8ad8", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 23.49, "tpot_relative": 1.2349, "ttft_error_pct": 43.32, "ttft_relative": 1.433161 }, "aisimulate": { "tpot_error_pct": 20.0, "tpot_relative": 1.200033, "ttft_error_pct": 5.61, "ttft_relative": 0.943922 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 33.31, "tpot_relative": 1.558472, "ttft_error_pct": 24.3, "ttft_relative": 1.577336 }, "aisimulate": { "tpot_error_pct": 29.77, "tpot_relative": 1.517075, "ttft_error_pct": 8.11, "ttft_relative": 1.166109 }, "concurrency": 8, "measured": { "tpot_relative": 1.169025, "ttft_relative": 1.268978 }, "status": "success" }, { "aic": { "tpot_error_pct": 36.97, "tpot_relative": 2.105592, "ttft_error_pct": 2.81, "ttft_relative": 1.725508 }, "aisimulate": { "tpot_error_pct": 33.85, "tpot_relative": 2.057684, "ttft_error_pct": 11.86, "ttft_relative": 1.47926 }, "concurrency": 16, "measured": { "tpot_relative": 1.537291, "ttft_relative": 1.678374 }, "status": "success" }, { "aic": { "tpot_error_pct": 41.88, "tpot_relative": 2.96353, "ttft_error_pct": 22.51, "ttft_relative": 1.877805 }, "aisimulate": { "tpot_error_pct": 39.53, "tpot_relative": 2.914415, "ttft_error_pct": 10.5, "ttft_relative": 2.168766 }, "concurrency": 32, "measured": { "tpot_relative": 2.088727, "ttft_relative": 2.42319 }, "status": "success" }, { "aic": { "tpot_error_pct": 50.09, "tpot_relative": 4.587321, "ttft_error_pct": 39.33, "ttft_relative": 2.039555 }, "aisimulate": { "tpot_error_pct": 48.84, "tpot_relative": 4.548978, "ttft_error_pct": 6.01, "ttft_relative": 3.563985 }, "concurrency": 64, "measured": { "tpot_relative": 3.056317, "ttft_relative": 3.361802 }, "status": "success" }, { "aic": { "tpot_error_pct": 53.57, "tpot_relative": 7.263849, "ttft_error_pct": 61.38, "ttft_relative": 2.227967 }, "aisimulate": { "tpot_error_pct": 53.11, "tpot_relative": 7.241745, "ttft_error_pct": 12.76, "ttft_relative": 6.505651 }, "concurrency": 128, "measured": { "tpot_relative": 4.729896, "ttft_relative": 5.769401 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 15, "tpot_mape_pct": 10.34, "tpot_shape_error_pct": 7.07, "ttft_mape_pct": 45.43, "ttft_shape_error_pct": 32.31 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.47, "tpot_shape_error_pct": 6.35, "ttft_mape_pct": 50.18, "ttft_shape_error_pct": 10.69 }, "gpu": "h100", "precisions": [ "fp4" ], "rows": 15, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 5.9, "tpot_shape_error_pct": 6.62, "ttft_mape_pct": 50.34, "ttft_shape_error_pct": 35.37 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.84, "tpot_shape_error_pct": 5.54, "ttft_mape_pct": 50.56, "ttft_shape_error_pct": 2.07 }, "framework": "vllm", "id": "f8ceae9a704b9214", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 1.09, "tpot_relative": 0.989102, "ttft_error_pct": 30.74, "ttft_relative": 0.692562 }, "aisimulate": { "tpot_error_pct": 4.05, "tpot_relative": 0.95947, "ttft_error_pct": 50.37, "ttft_relative": 0.49627 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.25, "tpot_relative": 1.357634, "ttft_error_pct": 40.63, "ttft_relative": 0.763312 }, "aisimulate": { "tpot_error_pct": 0.4, "tpot_relative": 1.322444, "ttft_error_pct": 52.18, "ttft_relative": 0.614777 }, "concurrency": 8, "measured": { "tpot_relative": 1.327811, "ttft_relative": 1.285721 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.87, "tpot_relative": 1.858424, "ttft_error_pct": 46.71, "ttft_relative": 0.835582 }, "aisimulate": { "tpot_error_pct": 4.74, "tpot_relative": 1.822542, "ttft_error_pct": 50.22, "ttft_relative": 0.780451 }, "concurrency": 16, "measured": { "tpot_relative": 1.913272, "ttft_relative": 1.567905 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.95, "tpot_relative": 2.57393, "ttft_error_pct": 59.35, "ttft_relative": 0.910781 }, "aisimulate": { "tpot_error_pct": 13.27, "tpot_relative": 2.535475, "ttft_error_pct": 48.94, "ttft_relative": 1.14414 }, "concurrency": 32, "measured": { "tpot_relative": 2.923376, "ttft_relative": 2.240593 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.32, "tpot_relative": 4.155373, "ttft_error_pct": 74.26, "ttft_relative": 0.994115 }, "aisimulate": { "tpot_error_pct": 11.76, "tpot_relative": 4.135034, "ttft_error_pct": 51.09, "ttft_relative": 1.889205 }, "concurrency": 64, "measured": { "tpot_relative": 4.685947, "ttft_relative": 3.862551 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 8.42, "tpot_shape_error_pct": 3.49, "ttft_mape_pct": 45.62, "ttft_shape_error_pct": 38.1 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.81, "tpot_shape_error_pct": 2.86, "ttft_mape_pct": 51.05, "ttft_shape_error_pct": 8.29 }, "framework": "vllm", "id": "460abc9ecca42a44", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 7.38, "tpot_relative": 0.926242, "ttft_error_pct": 21.77, "ttft_relative": 0.782312 }, "aisimulate": { "tpot_error_pct": 9.48, "tpot_relative": 0.905245, "ttft_error_pct": 48.54, "ttft_relative": 0.514616 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.15, "tpot_relative": 1.190293, "ttft_error_pct": 27.26, "ttft_relative": 0.861227 }, "aisimulate": { "tpot_error_pct": 6.2, "tpot_relative": 1.164851, "ttft_error_pct": 46.28, "ttft_relative": 0.636038 }, "concurrency": 8, "measured": { "tpot_relative": 1.241865, "ttft_relative": 1.184029 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.76, "tpot_relative": 1.570886, "ttft_error_pct": 45.93, "ttft_relative": 0.942409 }, "aisimulate": { "tpot_error_pct": 8.42, "tpot_relative": 1.542977, "ttft_error_pct": 53.81, "ttft_relative": 0.805112 }, "concurrency": 16, "measured": { "tpot_relative": 1.684823, "ttft_relative": 1.742935 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.38, "tpot_relative": 2.183283, "ttft_error_pct": 61.13, "ttft_relative": 1.025953 }, "aisimulate": { "tpot_error_pct": 10.36, "tpot_relative": 2.159788, "ttft_error_pct": 55.3, "ttft_relative": 1.179948 }, "concurrency": 32, "measured": { "tpot_relative": 2.40933, "ttft_relative": 2.639647 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.45, "tpot_relative": 3.254931, "ttft_error_pct": 71.98, "ttft_relative": 1.114802 }, "aisimulate": { "tpot_error_pct": 14.6, "tpot_relative": 3.248963, "ttft_error_pct": 51.31, "ttft_relative": 1.937254 }, "concurrency": 64, "measured": { "tpot_relative": 3.804559, "ttft_relative": 3.978947 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 16.7, "tpot_shape_error_pct": 11.1, "ttft_mape_pct": 40.34, "ttft_shape_error_pct": 23.47 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.75, "tpot_shape_error_pct": 10.67, "ttft_mape_pct": 48.92, "ttft_shape_error_pct": 21.71 }, "framework": "vllm", "id": "90f85a633bbc2797", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 8.73, "tpot_relative": 0.912658, "ttft_error_pct": 27.81, "ttft_relative": 0.721913 }, "aisimulate": { "tpot_error_pct": 10.38, "tpot_relative": 0.896205, "ttft_error_pct": 56.48, "ttft_relative": 0.435205 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.96, "tpot_relative": 1.04232, "ttft_error_pct": 25.24, "ttft_relative": 0.794509 }, "aisimulate": { "tpot_error_pct": 10.53, "tpot_relative": 1.024369, "ttft_error_pct": 49.54, "ttft_relative": 0.5363 }, "concurrency": 8, "measured": { "tpot_relative": 1.144947, "ttft_relative": 1.062808 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.38, "tpot_relative": 1.348223, "ttft_error_pct": 36.57, "ttft_relative": 0.867925 }, "aisimulate": { "tpot_error_pct": 9.7, "tpot_relative": 1.328871, "ttft_error_pct": 50.42, "ttft_relative": 0.678395 }, "concurrency": 16, "measured": { "tpot_relative": 1.471544, "ttft_relative": 1.368212 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.35, "tpot_relative": 1.765603, "ttft_error_pct": 48.04, "ttft_relative": 0.944313 }, "aisimulate": { "tpot_error_pct": 31.1, "tpot_relative": 1.746473, "ttft_error_pct": 45.39, "ttft_relative": 0.992438 }, "concurrency": 32, "measured": { "tpot_relative": 2.534793, "ttft_relative": 1.817212 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.08, "tpot_relative": 2.637067, "ttft_error_pct": 64.04, "ttft_relative": 1.024058 }, "aisimulate": { "tpot_error_pct": 27.07, "tpot_relative": 2.637402, "ttft_error_pct": 42.77, "ttft_relative": 1.629763 }, "concurrency": 64, "measured": { "tpot_relative": 3.61617, "ttft_relative": 2.847923 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 25, "tpot_mape_pct": 11.66, "tpot_shape_error_pct": 8.76, "ttft_mape_pct": 54.59, "ttft_shape_error_pct": 40.34 }, "aisimulate": { "coverage_pct": 100.0, "points": 25, "status_counts": { "failed": 0, "success": 25, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.69, "tpot_shape_error_pct": 7.55, "ttft_mape_pct": 59.99, "ttft_shape_error_pct": 25.34 }, "gpu": "h200", "precisions": [ "fp4" ], "rows": 25, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 17.17, "tpot_shape_error_pct": 10.2, "ttft_mape_pct": 48.94, "ttft_shape_error_pct": 7.45 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.23, "tpot_shape_error_pct": 6.45, "ttft_mape_pct": 64.6, "ttft_shape_error_pct": 41.83 }, "framework": "trtllm", "id": "37f8d0b3842d748a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 23.42, "tpot_relative": 0.76579, "ttft_error_pct": 50.22, "ttft_relative": 0.497793 }, "aisimulate": { "tpot_error_pct": 19.39, "tpot_relative": 0.806138, "ttft_error_pct": 73.48, "ttft_relative": 0.265234 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.55, "tpot_relative": 1.174769, "ttft_error_pct": 50.89, "ttft_relative": 0.547773 }, "aisimulate": { "tpot_error_pct": 15.98, "tpot_relative": 1.211852, "ttft_error_pct": 70.61, "ttft_relative": 0.327792 }, "concurrency": 8, "measured": { "tpot_relative": 1.442349, "ttft_relative": 1.115466 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.36, "tpot_relative": 1.686263, "ttft_error_pct": 44.41, "ttft_relative": 0.646976 }, "aisimulate": { "tpot_error_pct": 15.67, "tpot_relative": 1.720869, "ttft_error_pct": 64.28, "ttft_relative": 0.415661 }, "concurrency": 16, "measured": { "tpot_relative": 2.040567, "ttft_relative": 1.163741 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.13, "tpot_relative": 2.414554, "ttft_error_pct": 45.43, "ttft_relative": 0.849693 }, "aisimulate": { "tpot_error_pct": 16.33, "tpot_relative": 2.437917, "ttft_error_pct": 60.99, "ttft_relative": 0.607504 }, "concurrency": 32, "measured": { "tpot_relative": 2.913809, "ttft_relative": 1.557121 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.4, "tpot_relative": 4.064033, "ttft_error_pct": 53.77, "ttft_relative": 0.995875 }, "aisimulate": { "tpot_error_pct": 8.77, "tpot_relative": 4.092387, "ttft_error_pct": 53.65, "ttft_relative": 0.998521 }, "concurrency": 64, "measured": { "tpot_relative": 4.485651, "ttft_relative": 2.15432 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 12.27, "tpot_shape_error_pct": 5.63, "ttft_mape_pct": 46.29, "ttft_shape_error_pct": 34.05 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.21, "tpot_shape_error_pct": 5.08, "ttft_mape_pct": 55.6, "ttft_shape_error_pct": 6.08 }, "framework": "vllm", "id": "74714ee81fbee1ff", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 8.19, "tpot_relative": 0.918069, "ttft_error_pct": 26.19, "ttft_relative": 0.738076 }, "aisimulate": { "tpot_error_pct": 9.77, "tpot_relative": 0.902258, "ttft_error_pct": 56.37, "ttft_relative": 0.436261 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.75, "tpot_relative": 1.038011, "ttft_error_pct": 33.25, "ttft_relative": 0.81235 }, "aisimulate": { "tpot_error_pct": 11.2, "tpot_relative": 1.021324, "ttft_error_pct": 55.86, "ttft_relative": 0.537229 }, "concurrency": 8, "measured": { "tpot_relative": 1.15016, "ttft_relative": 1.217049 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.06, "tpot_relative": 1.340799, "ttft_error_pct": 44.57, "ttft_relative": 0.88737 }, "aisimulate": { "tpot_error_pct": 9.18, "tpot_relative": 1.324453, "ttft_error_pct": 57.53, "ttft_relative": 0.679819 }, "concurrency": 16, "measured": { "tpot_relative": 1.458322, "ttft_relative": 1.600798 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.03, "tpot_relative": 1.708302, "ttft_error_pct": 59.79, "ttft_relative": 0.964593 }, "aisimulate": { "tpot_error_pct": 13.71, "tpot_relative": 1.69499, "ttft_error_pct": 58.59, "ttft_relative": 0.993274 }, "concurrency": 32, "measured": { "tpot_relative": 1.964323, "ttft_relative": 2.398815 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.33, "tpot_relative": 2.421909, "ttft_error_pct": 67.67, "ttft_relative": 1.044707 }, "aisimulate": { "tpot_error_pct": 22.16, "tpot_relative": 2.427192, "ttft_error_pct": 49.66, "ttft_relative": 1.626794 }, "concurrency": 64, "measured": { "tpot_relative": 3.118349, "ttft_relative": 3.231685 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 7.21, "tpot_shape_error_pct": 8.98, "ttft_mape_pct": 69.13, "ttft_shape_error_pct": 70.27 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.32, "tpot_shape_error_pct": 8.67, "ttft_mape_pct": 69.17, "ttft_shape_error_pct": 48.64 }, "framework": "vllm", "id": "ee46f3d496f49b55", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 0.1, "tpot_relative": 1.001036, "ttft_error_pct": 29.5, "ttft_relative": 0.704959 }, "aisimulate": { "tpot_error_pct": 2.88, "tpot_relative": 0.97116, "ttft_error_pct": 49.87, "ttft_relative": 0.501326 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.26, "tpot_relative": 1.340055, "ttft_error_pct": 97.84, "ttft_relative": 0.776561 }, "aisimulate": { "tpot_error_pct": 7.46, "tpot_relative": 1.30602, "ttft_error_pct": 98.28, "ttft_relative": 0.620323 }, "concurrency": 8, "measured": { "tpot_relative": 1.215368, "ttft_relative": 36.001974 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.37, "tpot_relative": 1.854916, "ttft_error_pct": 47.38, "ttft_relative": 0.849455 }, "aisimulate": { "tpot_error_pct": 5.18, "tpot_relative": 1.82, "ttft_error_pct": 51.21, "ttft_relative": 0.787652 }, "concurrency": 16, "measured": { "tpot_relative": 1.919515, "ttft_relative": 1.61435 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.93, "tpot_relative": 2.56688, "ttft_error_pct": 97.7, "ttft_relative": 0.92462 }, "aisimulate": { "tpot_error_pct": 9.47, "tpot_relative": 2.533071, "ttft_error_pct": 97.13, "ttft_relative": 1.154423 }, "concurrency": 32, "measured": { "tpot_relative": 2.313997, "ttft_relative": 40.282527 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.4, "tpot_relative": 4.127636, "ttft_error_pct": 73.23, "ttft_relative": 1.006338 }, "aisimulate": { "tpot_error_pct": 11.58, "tpot_relative": 4.118974, "ttft_error_pct": 49.34, "ttft_relative": 1.904624 }, "concurrency": 64, "measured": { "tpot_relative": 4.658601, "ttft_relative": 3.759842 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 10.7, "tpot_shape_error_pct": 13.02, "ttft_mape_pct": 60.92, "ttft_shape_error_pct": 52.04 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.54, "tpot_shape_error_pct": 12.53, "ttft_mape_pct": 57.91, "ttft_shape_error_pct": 26.0 }, "framework": "vllm", "id": "8b88045544c67c3a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 0.94, "tpot_relative": 1.009418, "ttft_error_pct": 33.04, "ttft_relative": 0.669564 }, "aisimulate": { "tpot_error_pct": 2.74, "tpot_relative": 0.972628, "ttft_error_pct": 49.02, "ttft_relative": 0.509758 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.27, "tpot_relative": 1.490074, "ttft_error_pct": 38.45, "ttft_relative": 0.737826 }, "aisimulate": { "tpot_error_pct": 0.61, "tpot_relative": 1.448004, "ttft_error_pct": 47.28, "ttft_relative": 0.631897 }, "concurrency": 8, "measured": { "tpot_relative": 1.456944, "ttft_relative": 1.198668 }, "status": "success" }, { "aic": { "tpot_error_pct": 26.12, "tpot_relative": 2.147257, "ttft_error_pct": 96.93, "ttft_relative": 0.807488 }, "aisimulate": { "tpot_error_pct": 23.45, "tpot_relative": 2.101934, "ttft_error_pct": 96.94, "ttft_relative": 0.804277 }, "concurrency": 16, "measured": { "tpot_relative": 1.702609, "ttft_relative": 26.300559 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.29, "tpot_relative": 3.08902, "ttft_error_pct": 62.56, "ttft_relative": 0.880906 }, "aisimulate": { "tpot_error_pct": 12.64, "tpot_relative": 3.041969, "ttft_error_pct": 49.82, "ttft_relative": 1.180606 }, "concurrency": 32, "measured": { "tpot_relative": 3.482061, "ttft_relative": 2.352972 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.9, "tpot_relative": 5.097354, "ttft_error_pct": 73.6, "ttft_relative": 0.96323 }, "aisimulate": { "tpot_error_pct": 13.27, "tpot_relative": 5.075267, "ttft_error_pct": 46.46, "ttft_relative": 1.953677 }, "concurrency": 64, "measured": { "tpot_relative": 5.852006, "ttft_relative": 3.64929 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 10.93, "tpot_shape_error_pct": 6.0, "ttft_mape_pct": 47.68, "ttft_shape_error_pct": 37.89 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.17, "tpot_shape_error_pct": 5.03, "ttft_mape_pct": 52.69, "ttft_shape_error_pct": 4.15 }, "framework": "vllm", "id": "020482faed72950f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.65, "tpot_relative": 0.913519, "ttft_error_pct": 24.92, "ttft_relative": 0.750774 }, "aisimulate": { "tpot_error_pct": 10.67, "tpot_relative": 0.89327, "ttft_error_pct": 51.07, "ttft_relative": 0.489335 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.39, "tpot_relative": 1.158449, "ttft_error_pct": 37.99, "ttft_relative": 0.826648 }, "aisimulate": { "tpot_error_pct": 5.46, "tpot_relative": 1.133598, "ttft_error_pct": 54.68, "ttft_relative": 0.604197 }, "concurrency": 8, "measured": { "tpot_relative": 1.199043, "ttft_relative": 1.333038 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.22, "tpot_relative": 1.473482, "ttft_error_pct": 46.74, "ttft_relative": 0.903809 }, "aisimulate": { "tpot_error_pct": 10.64, "tpot_relative": 1.450574, "ttft_error_pct": 55.02, "ttft_relative": 0.763418 }, "concurrency": 16, "measured": { "tpot_relative": 1.62321, "ttft_relative": 1.697117 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.15, "tpot_relative": 1.966885, "ttft_error_pct": 56.98, "ttft_relative": 0.982959 }, "aisimulate": { "tpot_error_pct": 16.97, "tpot_relative": 1.947617, "ttft_error_pct": 51.19, "ttft_relative": 1.115439 }, "concurrency": 32, "measured": { "tpot_relative": 2.345687, "ttft_relative": 2.285142 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.22, "tpot_relative": 3.06451, "ttft_error_pct": 71.77, "ttft_relative": 1.066365 }, "aisimulate": { "tpot_error_pct": 17.1, "tpot_relative": 3.069201, "ttft_error_pct": 51.5, "ttft_relative": 1.832247 }, "concurrency": 64, "measured": { "tpot_relative": 3.702208, "ttft_relative": 3.777864 }, "status": "success" } ], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4" ], "rows": 69 } ] }, { "aic": { "points": 51, "tpot_mape_pct": 35.4, "tpot_shape_error_pct": 15.32, "ttft_mape_pct": 37.66, "ttft_shape_error_pct": 30.2 }, "aisimulate": { "coverage_pct": 100.0, "points": 51, "status_counts": { "failed": 0, "success": 51, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 31.64, "tpot_shape_error_pct": 14.73, "ttft_mape_pct": 11.0, "ttft_shape_error_pct": 6.67 }, "frameworks": [ "vllm" ], "gpu_skus": [ "b200", "b300" ], "hf_model_paths": [ "moonshotai/Kimi-K2.5" ], "model": "Kimi-K2.5", "precisions": [ "fp4" ], "rows": 51, "workloads": [ { "aic": { "points": 51, "tpot_mape_pct": 35.4, "tpot_shape_error_pct": 15.32, "ttft_mape_pct": 37.66, "ttft_shape_error_pct": 30.2 }, "aisimulate": { "coverage_pct": 100.0, "points": 51, "status_counts": { "failed": 0, "success": 51, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 31.64, "tpot_shape_error_pct": 14.73, "ttft_mape_pct": 11.0, "ttft_shape_error_pct": 6.67 }, "gpu_skus": [ "b200", "b300" ], "gpus": [ { "aic": { "points": 30, "tpot_mape_pct": 27.64, "tpot_shape_error_pct": 18.74, "ttft_mape_pct": 32.18, "ttft_shape_error_pct": 27.41 }, "aisimulate": { "coverage_pct": 100.0, "points": 30, "status_counts": { "failed": 0, "success": 30, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 22.11, "tpot_shape_error_pct": 17.35, "ttft_mape_pct": 9.36, "ttft_shape_error_pct": 8.63 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 30, "topologies": [ { "aic": { "points": 8, "tpot_mape_pct": 33.68, "tpot_shape_error_pct": 29.16, "ttft_mape_pct": 35.44, "ttft_shape_error_pct": 38.31 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 32.2, "tpot_shape_error_pct": 29.61, "ttft_mape_pct": 19.19, "ttft_shape_error_pct": 19.56 }, "framework": "vllm", "id": "aadedb69a2970221", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 9.26, "tpot_relative": 1.092634, "ttft_error_pct": 31.62, "ttft_relative": 1.316193 }, "aisimulate": { "tpot_error_pct": 8.88, "tpot_relative": 1.088816, "ttft_error_pct": 2.51, "ttft_relative": 0.97495 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.01, "tpot_relative": 1.190602, "ttft_error_pct": 20.81, "ttft_relative": 1.48086 }, "aisimulate": { "tpot_error_pct": 4.71, "tpot_relative": 1.154248, "ttft_error_pct": 16.85, "ttft_relative": 1.019199 }, "concurrency": 2, "measured": { "tpot_relative": 1.10228, "ttft_relative": 1.225743 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.95, "tpot_relative": 1.353949, "ttft_error_pct": 31.66, "ttft_relative": 1.645716 }, "aisimulate": { "tpot_error_pct": 2.37, "tpot_relative": 1.30812, "ttft_error_pct": 9.19, "ttft_relative": 1.135166 }, "concurrency": 4, "measured": { "tpot_relative": 1.277889, "ttft_relative": 1.250003 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.51, "tpot_relative": 1.959467, "ttft_error_pct": 7.63, "ttft_relative": 1.810984 }, "aisimulate": { "tpot_error_pct": 22.0, "tpot_relative": 1.90476, "ttft_error_pct": 15.71, "ttft_relative": 1.418233 }, "concurrency": 8, "measured": { "tpot_relative": 1.56122, "ttft_relative": 1.682553 }, "status": "success" }, { "aic": { "tpot_error_pct": 54.6, "tpot_relative": 3.150463, "ttft_error_pct": 0.06, "ttft_relative": 1.977137 }, "aisimulate": { "tpot_error_pct": 52.0, "tpot_relative": 3.09756, "ttft_error_pct": 7.51, "ttft_relative": 1.827494 }, "concurrency": 16, "measured": { "tpot_relative": 2.037814, "ttft_relative": 1.975965 }, "status": "success" }, { "aic": { "tpot_error_pct": 52.06, "tpot_relative": 4.247826, "ttft_error_pct": 24.55, "ttft_relative": 2.14519 }, "aisimulate": { "tpot_error_pct": 50.87, "tpot_relative": 4.214655, "ttft_error_pct": 4.9, "ttft_relative": 2.703598 }, "concurrency": 32, "measured": { "tpot_relative": 2.793528, "ttft_relative": 2.843042 }, "status": "success" }, { "aic": { "tpot_error_pct": 92.76, "tpot_relative": 5.636906, "ttft_error_pct": 96.87, "ttft_relative": 2.317296 }, "aisimulate": { "tpot_error_pct": 92.74, "tpot_relative": 5.647797, "ttft_error_pct": 93.98, "ttft_relative": 4.457882 }, "concurrency": 64, "measured": { "tpot_relative": 77.832123, "ttft_relative": 74.019761 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.3, "tpot_relative": 8.431865, "ttft_error_pct": 70.33, "ttft_relative": 2.498021 }, "aisimulate": { "tpot_error_pct": 24.0, "tpot_relative": 8.619977, "ttft_error_pct": 2.86, "ttft_relative": 8.179139 }, "concurrency": 128, "measured": { "tpot_relative": 6.951486, "ttft_relative": 8.419665 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 27.71, "tpot_shape_error_pct": 6.78, "ttft_mape_pct": 71.0, "ttft_shape_error_pct": 40.16 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.14, "tpot_shape_error_pct": 4.2, "ttft_mape_pct": 12.16, "ttft_shape_error_pct": 8.39 }, "framework": "vllm", "id": "3b097360b254b5bf", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 30.08, "tpot_relative": 0.699173, "ttft_error_pct": 63.72, "ttft_relative": 0.362814 }, "aisimulate": { "tpot_error_pct": 7.4, "tpot_relative": 1.07399, "ttft_error_pct": 15.7, "ttft_relative": 0.842999 }, "concurrency": 512, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.34, "tpot_relative": 1.194158, "ttft_error_pct": 78.29, "ttft_relative": 0.391655 }, "aisimulate": { "tpot_error_pct": 2.89, "tpot_relative": 1.645686, "ttft_error_pct": 8.63, "ttft_relative": 1.648231 }, "concurrency": 1024, "measured": { "tpot_relative": 1.599487, "ttft_relative": 1.803835 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 19.76, "tpot_shape_error_pct": 13.71, "ttft_mape_pct": 20.79, "ttft_shape_error_pct": 15.78 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.1, "tpot_shape_error_pct": 11.02, "ttft_mape_pct": 8.0, "ttft_shape_error_pct": 2.05 }, "framework": "vllm", "id": "a8509e8b6e40f124", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.53, "tpot_relative": 1.085306, "ttft_error_pct": 24.18, "ttft_relative": 1.241784 }, "aisimulate": { "tpot_error_pct": 4.56, "tpot_relative": 1.045584, "ttft_error_pct": 6.4, "ttft_relative": 0.935958 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.02, "tpot_relative": 1.194457, "ttft_error_pct": 29.95, "ttft_relative": 1.39713 }, "aisimulate": { "tpot_error_pct": 1.63, "tpot_relative": 1.118878, "ttft_error_pct": 9.05, "ttft_relative": 0.977757 }, "concurrency": 2, "measured": { "tpot_relative": 1.137367, "ttft_relative": 1.075106 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.0, "tpot_relative": 1.492966, "ttft_error_pct": 29.31, "ttft_relative": 1.552643 }, "aisimulate": { "tpot_error_pct": 2.96, "tpot_relative": 1.372462, "ttft_error_pct": 9.1, "ttft_relative": 1.091524 }, "concurrency": 4, "measured": { "tpot_relative": 1.333019, "ttft_relative": 1.200733 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.23, "tpot_relative": 2.12897, "ttft_error_pct": 13.41, "ttft_relative": 1.708513 }, "aisimulate": { "tpot_error_pct": 17.62, "tpot_relative": 1.968156, "ttft_error_pct": 9.44, "ttft_relative": 1.364291 }, "concurrency": 8, "measured": { "tpot_relative": 1.673316, "ttft_relative": 1.506514 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.15, "tpot_relative": 2.911481, "ttft_error_pct": 0.4, "ttft_relative": 1.865155 }, "aisimulate": { "tpot_error_pct": 21.21, "tpot_relative": 2.711577, "ttft_error_pct": 6.71, "ttft_relative": 1.746895 }, "concurrency": 16, "measured": { "tpot_relative": 2.237092, "ttft_relative": 1.872629 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.64, "tpot_relative": 4.298007, "ttft_error_pct": 27.52, "ttft_relative": 2.023452 }, "aisimulate": { "tpot_error_pct": 24.66, "tpot_relative": 3.950008, "ttft_error_pct": 7.29, "ttft_relative": 2.588149 }, "concurrency": 32, "measured": { "tpot_relative": 3.168719, "ttft_relative": 2.79167 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 8, "tpot_mape_pct": 30.43, "tpot_shape_error_pct": 19.79, "ttft_mape_pct": 35.01, "ttft_shape_error_pct": 31.01 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 28.94, "tpot_shape_error_pct": 18.72, "ttft_mape_pct": 2.61, "ttft_shape_error_pct": 4.88 }, "framework": "vllm", "id": "2cd98741fc7c6c22", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 11.18, "tpot_relative": 1.111774, "ttft_error_pct": 37.81, "ttft_relative": 1.378121 }, "aisimulate": { "tpot_error_pct": 10.79, "tpot_relative": 1.107879, "ttft_error_pct": 4.42, "ttft_relative": 1.044214 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.44, "tpot_relative": 1.227356, "ttft_error_pct": 34.79, "ttft_relative": 1.550528 }, "aisimulate": { "tpot_error_pct": 11.72, "tpot_relative": 1.187839, "ttft_error_pct": 5.03, "ttft_relative": 1.09246 }, "concurrency": 2, "measured": { "tpot_relative": 1.063239, "ttft_relative": 1.150291 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.4, "tpot_relative": 1.42784, "ttft_error_pct": 36.72, "ttft_relative": 1.723122 }, "aisimulate": { "tpot_error_pct": 11.42, "tpot_relative": 1.378648, "ttft_error_pct": 3.57, "ttft_relative": 1.215364 }, "concurrency": 4, "measured": { "tpot_relative": 1.237339, "ttft_relative": 1.260364 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.52, "tpot_relative": 1.965548, "ttft_error_pct": 24.59, "ttft_relative": 1.896125 }, "aisimulate": { "tpot_error_pct": 23.81, "tpot_relative": 1.9084, "ttft_error_pct": 0.67, "ttft_relative": 1.511663 }, "concurrency": 8, "measured": { "tpot_relative": 1.541353, "ttft_relative": 1.521835 }, "status": "success" }, { "aic": { "tpot_error_pct": 55.83, "tpot_relative": 3.167536, "ttft_error_pct": 7.8, "ttft_relative": 2.070006 }, "aisimulate": { "tpot_error_pct": 53.15, "tpot_relative": 3.113078, "ttft_error_pct": 1.03, "ttft_relative": 1.940108 }, "concurrency": 16, "measured": { "tpot_relative": 2.032689, "ttft_relative": 1.920272 }, "status": "success" }, { "aic": { "tpot_error_pct": 52.12, "tpot_relative": 4.379733, "ttft_error_pct": 21.81, "ttft_relative": 2.245771 }, "aisimulate": { "tpot_error_pct": 50.97, "tpot_relative": 4.346801, "ttft_error_pct": 0.25, "ttft_relative": 2.864925 }, "concurrency": 32, "measured": { "tpot_relative": 2.879175, "ttft_relative": 2.872193 }, "status": "success" }, { "aic": { "tpot_error_pct": 40.26, "tpot_relative": 6.043647, "ttft_error_pct": 48.43, "ttft_relative": 2.425555 }, "aisimulate": { "tpot_error_pct": 40.81, "tpot_relative": 6.067639, "ttft_error_pct": 0.39, "ttft_relative": 4.721529 }, "concurrency": 64, "measured": { "tpot_relative": 4.309019, "ttft_relative": 4.702979 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.74, "tpot_relative": 9.025491, "ttft_error_pct": 68.13, "ttft_relative": 2.613888 }, "aisimulate": { "tpot_error_pct": 28.83, "tpot_relative": 9.247591, "ttft_error_pct": 5.47, "ttft_relative": 8.651152 }, "concurrency": 128, "measured": { "tpot_relative": 7.177984, "ttft_relative": 8.202308 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 23.72, "tpot_shape_error_pct": 10.12, "ttft_mape_pct": 22.48, "ttft_shape_error_pct": 16.19 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.21, "tpot_shape_error_pct": 7.24, "ttft_mape_pct": 5.68, "ttft_shape_error_pct": 5.22 }, "framework": "vllm", "id": "a465cd48778fe1f6", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 14.8, "tpot_relative": 1.147996, "ttft_error_pct": 28.59, "ttft_relative": 1.285946 }, "aisimulate": { "tpot_error_pct": 10.67, "tpot_relative": 1.106683, "ttft_error_pct": 1.39, "ttft_relative": 0.986084 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.53, "tpot_relative": 1.268686, "ttft_error_pct": 29.51, "ttft_relative": 1.446817 }, "aisimulate": { "tpot_error_pct": 5.46, "tpot_relative": 1.188949, "ttft_error_pct": 7.73, "ttft_relative": 1.030841 }, "concurrency": 2, "measured": { "tpot_relative": 1.127437, "ttft_relative": 1.117165 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.89, "tpot_relative": 1.589596, "ttft_error_pct": 32.96, "ttft_relative": 1.607859 }, "aisimulate": { "tpot_error_pct": 9.47, "tpot_relative": 1.463539, "ttft_error_pct": 5.01, "ttft_relative": 1.148601 }, "concurrency": 4, "measured": { "tpot_relative": 1.336985, "ttft_relative": 1.209236 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.7, "tpot_relative": 2.123239, "ttft_error_pct": 15.39, "ttft_relative": 1.769271 }, "aisimulate": { "tpot_error_pct": 17.67, "tpot_relative": 1.956443, "ttft_error_pct": 6.85, "ttft_relative": 1.428244 }, "concurrency": 8, "measured": { "tpot_relative": 1.662712, "ttft_relative": 1.533337 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.41, "tpot_relative": 2.941877, "ttft_error_pct": 1.0, "ttft_relative": 1.931481 }, "aisimulate": { "tpot_error_pct": 21.24, "tpot_relative": 2.734965, "ttft_error_pct": 6.51, "ttft_relative": 1.824116 }, "concurrency": 16, "measured": { "tpot_relative": 2.255803, "ttft_relative": 1.951076 }, "status": "success" }, { "aic": { "tpot_error_pct": 38.01, "tpot_relative": 4.406035, "ttft_error_pct": 27.42, "ttft_relative": 2.095401 }, "aisimulate": { "tpot_error_pct": 26.75, "tpot_relative": 4.046549, "ttft_error_pct": 6.61, "ttft_relative": 2.696187 }, "concurrency": 32, "measured": { "tpot_relative": 3.192595, "ttft_relative": 2.887068 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 21, "tpot_mape_pct": 46.48, "tpot_shape_error_pct": 9.97, "ttft_mape_pct": 45.49, "ttft_shape_error_pct": 34.55 }, "aisimulate": { "coverage_pct": 100.0, "points": 21, "status_counts": { "failed": 0, "success": 21, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 45.25, "tpot_shape_error_pct": 10.63, "ttft_mape_pct": 13.34, "ttft_shape_error_pct": 3.61 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 21, "topologies": [ { "aic": { "points": 8, "tpot_mape_pct": 49.98, "tpot_shape_error_pct": 12.14, "ttft_mape_pct": 40.91, "ttft_shape_error_pct": 32.35 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 49.26, "tpot_shape_error_pct": 12.36, "ttft_mape_pct": 12.23, "ttft_shape_error_pct": 3.71 }, "framework": "vllm", "id": "2024936ff38d1bfd", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 35.58, "tpot_relative": 1.355795, "ttft_error_pct": 49.24, "ttft_relative": 1.492412 }, "aisimulate": { "tpot_error_pct": 34.69, "tpot_relative": 1.346949, "ttft_error_pct": 15.7, "ttft_relative": 1.156978 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 50.41, "tpot_relative": 1.543672, "ttft_error_pct": 53.3, "ttft_relative": 1.679113 }, "aisimulate": { "tpot_error_pct": 46.39, "tpot_relative": 1.502389, "ttft_error_pct": 10.73, "ttft_relative": 1.212864 }, "concurrency": 2, "measured": { "tpot_relative": 1.026287, "ttft_relative": 1.095331 }, "status": "success" }, { "aic": { "tpot_error_pct": 42.18, "tpot_relative": 1.84719, "ttft_error_pct": 53.36, "ttft_relative": 1.866015 }, "aisimulate": { "tpot_error_pct": 38.79, "tpot_relative": 1.803127, "ttft_error_pct": 11.39, "ttft_relative": 1.355282 }, "concurrency": 4, "measured": { "tpot_relative": 1.299197, "ttft_relative": 1.216726 }, "status": "success" }, { "aic": { "tpot_error_pct": 55.19, "tpot_relative": 2.448573, "ttft_error_pct": 31.01, "ttft_relative": 2.05335 }, "aisimulate": { "tpot_error_pct": 52.3, "tpot_relative": 2.403059, "ttft_error_pct": 8.14, "ttft_relative": 1.694892 }, "concurrency": 8, "measured": { "tpot_relative": 1.577801, "ttft_relative": 1.567295 }, "status": "success" }, { "aic": { "tpot_error_pct": 55.67, "tpot_relative": 3.29838, "ttft_error_pct": 12.9, "ttft_relative": 2.241618 }, "aisimulate": { "tpot_error_pct": 53.79, "tpot_relative": 3.258577, "ttft_error_pct": 9.74, "ttft_relative": 2.178896 }, "concurrency": 16, "measured": { "tpot_relative": 2.118813, "ttft_relative": 1.985559 }, "status": "success" }, { "aic": { "tpot_error_pct": 58.91, "tpot_relative": 4.651651, "ttft_error_pct": 16.05, "ttft_relative": 2.431888 }, "aisimulate": { "tpot_error_pct": 58.48, "tpot_relative": 4.638913, "ttft_error_pct": 11.76, "ttft_relative": 3.237418 }, "concurrency": 32, "measured": { "tpot_relative": 2.92722, "ttft_relative": 2.896855 }, "status": "success" }, { "aic": { "tpot_error_pct": 57.68, "tpot_relative": 6.785504, "ttft_error_pct": 44.66, "ttft_relative": 2.626431 }, "aisimulate": { "tpot_error_pct": 59.63, "tpot_relative": 6.869359, "ttft_error_pct": 13.54, "ttft_relative": 5.38877 }, "concurrency": 64, "measured": { "tpot_relative": 4.303246, "ttft_relative": 4.745981 }, "status": "success" }, { "aic": { "tpot_error_pct": 44.23, "tpot_relative": 9.521546, "ttft_error_pct": 66.75, "ttft_relative": 2.830066 }, "aisimulate": { "tpot_error_pct": 49.99, "tpot_relative": 9.902019, "ttft_error_pct": 16.83, "ttft_relative": 9.942844 }, "concurrency": 128, "measured": { "tpot_relative": 6.601625, "ttft_relative": 8.51071 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 44.88, "tpot_shape_error_pct": null, "ttft_mape_pct": 51.25, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 44.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 14.3, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "5c2735a1406fac02", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 44.88, "tpot_relative": 1.448825, "ttft_error_pct": 51.25, "ttft_relative": 1.512519 }, "aisimulate": { "tpot_error_pct": 44.28, "tpot_relative": 1.442829, "ttft_error_pct": 14.3, "ttft_relative": 1.143011 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 14.85, "tpot_shape_error_pct": 3.71, "ttft_mape_pct": 63.15, "ttft_shape_error_pct": 49.35 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.8, "tpot_shape_error_pct": 7.69, "ttft_mape_pct": 12.93, "ttft_shape_error_pct": 3.64 }, "framework": "vllm", "id": "06b1ef698a41051b", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 12.69, "tpot_relative": 0.873132, "ttft_error_pct": 45.07, "ttft_relative": 0.549254 }, "aisimulate": { "tpot_error_pct": 16.78, "tpot_relative": 1.167828, "ttft_error_pct": 15.0, "ttft_relative": 0.850021 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.51, "tpot_relative": 1.208164, "ttft_error_pct": 64.8, "ttft_relative": 0.59367 }, "aisimulate": { "tpot_error_pct": 10.1, "tpot_relative": 1.555894, "ttft_error_pct": 13.29, "ttft_relative": 1.462567 }, "concurrency": 256, "measured": { "tpot_relative": 1.413172, "ttft_relative": 1.686676 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.35, "tpot_relative": 1.846325, "ttft_error_pct": 79.56, "ttft_relative": 0.640724 }, "aisimulate": { "tpot_error_pct": 5.51, "tpot_relative": 2.356812, "ttft_error_pct": 10.52, "ttft_relative": 2.805343 }, "concurrency": 512, "measured": { "tpot_relative": 2.233795, "ttft_relative": 3.135002 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 8, "tpot_mape_pct": 54.3, "tpot_shape_error_pct": 9.59, "ttft_mape_pct": 41.81, "ttft_shape_error_pct": 32.53 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 53.49, "tpot_shape_error_pct": 9.74, "ttft_mape_pct": 13.89, "ttft_shape_error_pct": 3.5 }, "framework": "vllm", "id": "bee23d0c6f103d0f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 42.36, "tpot_relative": 1.423644, "ttft_error_pct": 49.89, "ttft_relative": 1.498883 }, "aisimulate": { "tpot_error_pct": 41.44, "tpot_relative": 1.414384, "ttft_error_pct": 17.49, "ttft_relative": 1.174884 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 50.08, "tpot_relative": 1.617695, "ttft_error_pct": 54.35, "ttft_relative": 1.686394 }, "aisimulate": { "tpot_error_pct": 45.97, "tpot_relative": 1.573439, "ttft_error_pct": 12.73, "ttft_relative": 1.231625 }, "concurrency": 2, "measured": { "tpot_relative": 1.077883, "ttft_relative": 1.092577 }, "status": "success" }, { "aic": { "tpot_error_pct": 54.14, "tpot_relative": 1.937498, "ttft_error_pct": 57.18, "ttft_relative": 1.874107 }, "aisimulate": { "tpot_error_pct": 50.39, "tpot_relative": 1.890383, "ttft_error_pct": 15.28, "ttft_relative": 1.374509 }, "concurrency": 4, "measured": { "tpot_relative": 1.257001, "ttft_relative": 1.192349 }, "status": "success" }, { "aic": { "tpot_error_pct": 63.29, "tpot_relative": 2.517306, "ttft_error_pct": 33.68, "ttft_relative": 2.062254 }, "aisimulate": { "tpot_error_pct": 60.15, "tpot_relative": 2.468991, "ttft_error_pct": 11.15, "ttft_relative": 1.71464 }, "concurrency": 8, "measured": { "tpot_relative": 1.541633, "ttft_relative": 1.542665 }, "status": "success" }, { "aic": { "tpot_error_pct": 62.27, "tpot_relative": 3.393097, "ttft_error_pct": 12.81, "ttft_relative": 2.251341 }, "aisimulate": { "tpot_error_pct": 60.26, "tpot_relative": 3.351008, "ttft_error_pct": 10.2, "ttft_relative": 2.199251 }, "concurrency": 16, "measured": { "tpot_relative": 2.090983, "ttft_relative": 1.995754 }, "status": "success" }, { "aic": { "tpot_error_pct": 59.04, "tpot_relative": 4.795586, "ttft_error_pct": 15.47, "ttft_relative": 2.442439 }, "aisimulate": { "tpot_error_pct": 58.56, "tpot_relative": 4.781174, "ttft_error_pct": 12.85, "ttft_relative": 3.260817 }, "concurrency": 32, "measured": { "tpot_relative": 3.015347, "ttft_relative": 2.889527 }, "status": "success" }, { "aic": { "tpot_error_pct": 56.58, "tpot_relative": 7.081164, "ttft_error_pct": 44.44, "ttft_relative": 2.637831 }, "aisimulate": { "tpot_error_pct": 58.52, "tpot_relative": 7.168772, "ttft_error_pct": 14.17, "ttft_relative": 5.420005 }, "concurrency": 64, "measured": { "tpot_relative": 4.522434, "ttft_relative": 4.747329 }, "status": "success" }, { "aic": { "tpot_error_pct": 46.68, "tpot_relative": 10.244621, "ttft_error_pct": 66.67, "ttft_relative": 2.842361 }, "aisimulate": { "tpot_error_pct": 52.61, "tpot_relative": 10.659236, "ttft_error_pct": 17.24, "ttft_relative": 9.997715 }, "concurrency": 128, "measured": { "tpot_relative": 6.98453, "ttft_relative": 8.527248 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 52.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 52.83, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 51.69, "tpot_shape_error_pct": null, "ttft_mape_pct": 18.02, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "4f53a66133a25243", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 52.32, "tpot_relative": 1.523201, "ttft_error_pct": 52.83, "ttft_relative": 1.528283 }, "aisimulate": { "tpot_error_pct": 51.69, "tpot_relative": 1.516887, "ttft_error_pct": 18.02, "ttft_relative": 1.18023 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4" ], "rows": 51 } ] }, { "aic": { "points": 168, "tpot_mape_pct": 15.39, "tpot_shape_error_pct": 6.78, "ttft_mape_pct": 49.59, "ttft_shape_error_pct": 35.67 }, "aisimulate": { "coverage_pct": 100.0, "points": 168, "status_counts": { "failed": 0, "success": 168, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.07, "tpot_shape_error_pct": 6.21, "ttft_mape_pct": 39.89, "ttft_shape_error_pct": 14.98 }, "frameworks": [ "dynamo-vllm", "trtllm", "vllm" ], "gpu_skus": [ "b200", "b300", "h200" ], "hf_model_paths": [ "MiniMaxAI/MiniMax-M2.5" ], "model": "MiniMax-M2.5", "precisions": [ "fp4", "fp8" ], "rows": 168, "workloads": [ { "aic": { "points": 80, "tpot_mape_pct": 15.14, "tpot_shape_error_pct": 7.1, "ttft_mape_pct": 56.5, "ttft_shape_error_pct": 29.56 }, "aisimulate": { "coverage_pct": 100.0, "points": 80, "status_counts": { "failed": 0, "success": 80, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.58, "tpot_shape_error_pct": 6.89, "ttft_mape_pct": 42.97, "ttft_shape_error_pct": 16.39 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 36, "tpot_mape_pct": 16.73, "tpot_shape_error_pct": 7.15, "ttft_mape_pct": 68.9, "ttft_shape_error_pct": 29.11 }, "aisimulate": { "coverage_pct": 100.0, "points": 36, "status_counts": { "failed": 0, "success": 36, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.87, "tpot_shape_error_pct": 7.54, "ttft_mape_pct": 36.28, "ttft_shape_error_pct": 13.28 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 36, "topologies": [ { "aic": { "points": 1, "tpot_mape_pct": 25.04, "tpot_shape_error_pct": null, "ttft_mape_pct": 38.67, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.26, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.71, "ttft_shape_error_pct": null }, "framework": "dynamo-vllm", "id": "5e100a74dc7cd4dd", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 25.04, "tpot_relative": 1.250445, "ttft_error_pct": 38.67, "ttft_relative": 0.613306 }, "aisimulate": { "tpot_error_pct": 15.26, "tpot_relative": 1.152609, "ttft_error_pct": 49.71, "ttft_relative": 0.502917 }, "concurrency": 16, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 21.27, "tpot_shape_error_pct": 4.8, "ttft_mape_pct": 70.17, "ttft_shape_error_pct": 51.51 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.08, "tpot_shape_error_pct": 4.72, "ttft_mape_pct": 56.22, "ttft_shape_error_pct": 6.96 }, "framework": "dynamo-vllm", "id": "7ac4fafec4dbb9d3", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 20.67, "tpot_relative": 1.206674, "ttft_error_pct": 51.4, "ttft_relative": 0.486034 }, "aisimulate": { "tpot_error_pct": 11.12, "tpot_relative": 1.11119, "ttft_error_pct": 53.81, "ttft_relative": 0.461916 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.42, "tpot_relative": 1.461511, "ttft_error_pct": 66.29, "ttft_relative": 0.486034 }, "aisimulate": { "tpot_error_pct": 17.1, "tpot_relative": 1.343192, "ttft_error_pct": 56.22, "ttft_relative": 0.631193 }, "concurrency": 64, "measured": { "tpot_relative": 1.147038, "ttft_relative": 1.441876 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.81, "tpot_relative": 1.654401, "ttft_error_pct": 77.29, "ttft_relative": 0.486034 }, "aisimulate": { "tpot_error_pct": 12.94, "tpot_relative": 1.509109, "ttft_error_pct": 57.65, "ttft_relative": 0.906328 }, "concurrency": 128, "measured": { "tpot_relative": 1.336221, "ttft_relative": 2.140036 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.2, "tpot_relative": 1.872639, "ttft_error_pct": 85.72, "ttft_relative": 0.486034 }, "aisimulate": { "tpot_error_pct": 3.17, "tpot_relative": 1.706731, "ttft_error_pct": 57.19, "ttft_relative": 1.456541 }, "concurrency": 256, "measured": { "tpot_relative": 1.654248, "ttft_relative": 3.402707 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 20.84, "tpot_shape_error_pct": 21.02, "ttft_mape_pct": 78.1, "ttft_shape_error_pct": 28.27 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.42, "tpot_shape_error_pct": 20.29, "ttft_mape_pct": 67.21, "ttft_shape_error_pct": 24.47 }, "framework": "dynamo-vllm", "id": "16ff450422ac8191", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 5.98, "tpot_relative": 1.059841, "ttft_error_pct": 73.02, "ttft_relative": 0.26985 }, "aisimulate": { "tpot_error_pct": 2.36, "tpot_relative": 0.976413, "ttft_error_pct": 71.81, "ttft_relative": 0.281882 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.3, "tpot_relative": 1.493268, "ttft_error_pct": 80.53, "ttft_relative": 0.26985 }, "aisimulate": { "tpot_error_pct": 17.86, "tpot_relative": 1.371804, "ttft_error_pct": 71.79, "ttft_relative": 0.390907 }, "concurrency": 64, "measured": { "tpot_relative": 1.163933, "ttft_relative": 1.385695 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.23, "tpot_relative": 1.753063, "ttft_error_pct": 80.76, "ttft_relative": 0.26985 }, "aisimulate": { "tpot_error_pct": 17.05, "tpot_relative": 1.600276, "ttft_error_pct": 58.04, "ttft_relative": 0.588546 }, "concurrency": 128, "measured": { "tpot_relative": 1.367162, "ttft_relative": 1.402605 }, "status": "success" } ], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 3.93, "tpot_shape_error_pct": 1.18, "ttft_mape_pct": 23.71, "ttft_shape_error_pct": 9.51 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.64, "tpot_shape_error_pct": 1.3, "ttft_mape_pct": 28.11, "ttft_shape_error_pct": 8.13 }, "framework": "trtllm", "id": "d3eb78f5cec0a457", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 4.54, "tpot_relative": 1.04541, "ttft_error_pct": 29.89, "ttft_relative": 1.298919 }, "aisimulate": { "tpot_error_pct": 5.33, "tpot_relative": 1.053267, "ttft_error_pct": 25.07, "ttft_relative": 0.749325 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.31, "tpot_relative": 1.132877, "ttft_error_pct": 17.54, "ttft_relative": 1.428518 }, "aisimulate": { "tpot_error_pct": 3.96, "tpot_relative": 1.13995, "ttft_error_pct": 31.16, "ttft_relative": 0.836647 }, "concurrency": 8, "measured": { "tpot_relative": 1.096557, "ttft_relative": 1.215382 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 11.01, "tpot_shape_error_pct": 3.11, "ttft_mape_pct": 51.32, "ttft_shape_error_pct": 0.02 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.82, "tpot_shape_error_pct": 3.09, "ttft_mape_pct": 11.84, "ttft_shape_error_pct": 2.36 }, "framework": "trtllm", "id": "fa843a80712f0ff0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 12.76, "tpot_relative": 1.127619, "ttft_error_pct": 51.33, "ttft_relative": 1.513287 }, "aisimulate": { "tpot_error_pct": 13.57, "tpot_relative": 1.135707, "ttft_error_pct": 12.87, "ttft_relative": 0.871287 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.26, "tpot_relative": 1.330331, "ttft_error_pct": 51.31, "ttft_relative": 1.663819 }, "aisimulate": { "tpot_error_pct": 10.06, "tpot_relative": 1.340125, "ttft_error_pct": 10.81, "ttft_relative": 0.980732 }, "concurrency": 8, "measured": { "tpot_relative": 1.217617, "ttft_relative": 1.099642 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 11.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 85.11, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.06, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.78, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "bbce71438bf42a72", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 11.32, "tpot_relative": 0.886778, "ttft_error_pct": 85.11, "ttft_relative": 0.148882 }, "aisimulate": { "tpot_error_pct": 12.06, "tpot_relative": 1.120649, "ttft_error_pct": 62.78, "ttft_relative": 0.372151 }, "concurrency": 1024, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 14.31, "tpot_shape_error_pct": 1.16, "ttft_mape_pct": 84.59, "ttft_shape_error_pct": 11.54 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.98, "tpot_shape_error_pct": 0.86, "ttft_mape_pct": 18.82, "ttft_shape_error_pct": 1.21 }, "framework": "vllm", "id": "5d464fcebfcd762a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 14.76, "tpot_relative": 1.147602, "ttft_error_pct": 99.97, "ttft_relative": 1.999678 }, "aisimulate": { "tpot_error_pct": 14.12, "tpot_relative": 1.14118, "ttft_error_pct": 18.16, "ttft_relative": 0.818413 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.42, "tpot_relative": 1.438581, "ttft_error_pct": 85.48, "ttft_relative": 2.201093 }, "aisimulate": { "tpot_error_pct": 14.9, "tpot_relative": 1.432029, "ttft_error_pct": 20.11, "ttft_relative": 0.948008 }, "concurrency": 8, "measured": { "tpot_relative": 1.246342, "ttft_relative": 1.186705 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.75, "tpot_relative": 1.796565, "ttft_error_pct": 68.31, "ttft_relative": 2.404335 }, "aisimulate": { "tpot_error_pct": 12.93, "tpot_relative": 1.799365, "ttft_error_pct": 18.18, "ttft_relative": 1.168858 }, "concurrency": 16, "measured": { "tpot_relative": 1.593367, "ttft_relative": 1.428514 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 22.25, "tpot_shape_error_pct": 5.23, "ttft_mape_pct": 108.87, "ttft_shape_error_pct": 9.64 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 22.26, "tpot_shape_error_pct": 4.84, "ttft_mape_pct": 29.28, "ttft_shape_error_pct": 1.83 }, "framework": "vllm", "id": "e49cff04761e76d6", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 26.67, "tpot_relative": 1.266672, "ttft_error_pct": 123.21, "ttft_relative": 2.232119 }, "aisimulate": { "tpot_error_pct": 26.34, "tpot_relative": 1.26338, "ttft_error_pct": 28.41, "ttft_relative": 0.715874 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.25, "tpot_relative": 1.384957, "ttft_error_pct": 114.5, "ttft_relative": 2.456196 }, "aisimulate": { "tpot_error_pct": 21.28, "tpot_relative": 1.385247, "ttft_error_pct": 28.96, "ttft_relative": 0.81349 }, "concurrency": 8, "measured": { "tpot_relative": 1.142213, "ttft_relative": 1.145072 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.82, "tpot_relative": 1.556488, "ttft_error_pct": 88.9, "ttft_relative": 2.681672 }, "aisimulate": { "tpot_error_pct": 19.18, "tpot_relative": 1.561137, "ttft_error_pct": 30.48, "ttft_relative": 0.986909 }, "concurrency": 16, "measured": { "tpot_relative": 1.309907, "ttft_relative": 1.419635 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 12.95, "tpot_shape_error_pct": 6.03, "ttft_mape_pct": 11.22, "ttft_shape_error_pct": 21.69 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.59, "tpot_shape_error_pct": 4.53, "ttft_mape_pct": 28.8, "ttft_shape_error_pct": 23.09 }, "framework": "vllm", "id": "d09679799df5b694", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 16.46, "tpot_relative": 1.164573, "ttft_error_pct": 3.51, "ttft_relative": 1.035101 }, "aisimulate": { "tpot_error_pct": 20.32, "tpot_relative": 1.20318, "ttft_error_pct": 36.17, "ttft_relative": 0.638319 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.44, "tpot_relative": 1.371239, "ttft_error_pct": 18.94, "ttft_relative": 1.125115 }, "aisimulate": { "tpot_error_pct": 14.87, "tpot_relative": 1.43928, "ttft_error_pct": 21.43, "ttft_relative": 1.090506 }, "concurrency": 128, "measured": { "tpot_relative": 1.253009, "ttft_relative": 1.387973 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 13.85, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.03, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.12, "tpot_shape_error_pct": null, "ttft_mape_pct": 21.68, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "239cc443fc3e2b53", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 13.85, "tpot_relative": 1.13851, "ttft_error_pct": 84.03, "ttft_relative": 1.840255 }, "aisimulate": { "tpot_error_pct": 14.12, "tpot_relative": 1.141157, "ttft_error_pct": 21.68, "ttft_relative": 0.7832 }, "concurrency": 16, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 24.5, "tpot_shape_error_pct": 0.22, "ttft_mape_pct": 138.5, "ttft_shape_error_pct": 3.55 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 24.24, "tpot_shape_error_pct": 0.2, "ttft_mape_pct": 28.98, "ttft_shape_error_pct": 0.28 }, "framework": "vllm", "id": "dcce887232ee48e0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 24.64, "tpot_relative": 1.246358, "ttft_error_pct": 142.82, "ttft_relative": 2.428182 }, "aisimulate": { "tpot_error_pct": 24.11, "tpot_relative": 1.241094, "ttft_error_pct": 28.88, "ttft_relative": 0.711176 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.36, "tpot_relative": 1.358281, "ttft_error_pct": 134.19, "ttft_relative": 2.673074 }, "aisimulate": { "tpot_error_pct": 24.36, "tpot_relative": 1.358269, "ttft_error_pct": 29.08, "ttft_relative": 0.809477 }, "concurrency": 8, "measured": { "tpot_relative": 1.092177, "ttft_relative": 1.141424 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 8.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 13.0, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.41, "tpot_shape_error_pct": null, "ttft_mape_pct": 1.7, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "8803d12c6daf4653", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 8.42, "tpot_relative": 1.084226, "ttft_error_pct": 13.0, "ttft_relative": 0.870029 }, "aisimulate": { "tpot_error_pct": 13.41, "tpot_relative": 1.134055, "ttft_error_pct": 1.7, "ttft_relative": 0.982957 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 9.87, "tpot_shape_error_pct": 7.14, "ttft_mape_pct": 68.55, "ttft_shape_error_pct": 96.74 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 3.28, "tpot_shape_error_pct": 5.35, "ttft_mape_pct": 58.93, "ttft_shape_error_pct": 94.54 }, "framework": "vllm", "id": "683a7505be3155f9", "parallelism": { "attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 6.54, "tpot_relative": 0.934634, "ttft_error_pct": 39.09, "ttft_relative": 0.609073 }, "aisimulate": { "tpot_error_pct": 6.11, "tpot_relative": 1.061142, "ttft_error_pct": 22.11, "ttft_relative": 0.778914 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.21, "tpot_relative": 1.251, "ttft_error_pct": 98.01, "ttft_relative": 0.693002 }, "aisimulate": { "tpot_error_pct": 0.44, "tpot_relative": 1.447739, "ttft_error_pct": 95.74, "ttft_relative": 1.484599 }, "concurrency": 512, "measured": { "tpot_relative": 1.441401, "ttft_relative": 34.878335 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 17.49, "tpot_shape_error_pct": 10.88, "ttft_mape_pct": 60.71, "ttft_shape_error_pct": 29.85 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 19.47, "tpot_shape_error_pct": 13.67, "ttft_mape_pct": 25.23, "ttft_shape_error_pct": 10.16 }, "framework": "vllm", "id": "1b1de4f7e5b7cd5f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 7.73, "tpot_relative": 1.077292, "ttft_error_pct": 106.52, "ttft_relative": 2.065223 }, "aisimulate": { "tpot_error_pct": 7.25, "tpot_relative": 1.072491, "ttft_error_pct": 18.32, "ttft_relative": 0.816837 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.46, "tpot_relative": 1.260078, "ttft_error_pct": 96.81, "ttft_relative": 2.273028 }, "aisimulate": { "tpot_error_pct": 12.25, "tpot_relative": 1.25765, "ttft_error_pct": 20.87, "ttft_relative": 0.913892 }, "concurrency": 8, "measured": { "tpot_relative": 1.120446, "ttft_relative": 1.154926 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.88, "tpot_relative": 1.527756, "ttft_error_pct": 83.3, "ttft_relative": 2.482709 }, "aisimulate": { "tpot_error_pct": 16.19, "tpot_relative": 1.531738, "ttft_error_pct": 22.61, "ttft_relative": 1.048166 }, "concurrency": 16, "measured": { "tpot_relative": 1.318341, "ttft_relative": 1.354434 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.4, "tpot_relative": 1.801015, "ttft_error_pct": 51.55, "ttft_relative": 2.695302 }, "aisimulate": { "tpot_error_pct": 11.74, "tpot_relative": 1.822732, "ttft_error_pct": 27.45, "ttft_relative": 1.290224 }, "concurrency": 32, "measured": { "tpot_relative": 1.631287, "ttft_relative": 1.778506 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.91, "tpot_relative": 2.599883, "ttft_error_pct": 9.39, "ttft_relative": 2.917078 }, "aisimulate": { "tpot_error_pct": 40.29, "tpot_relative": 2.683687, "ttft_error_pct": 32.52, "ttft_relative": 1.799501 }, "concurrency": 64, "measured": { "tpot_relative": 1.912921, "ttft_relative": 2.666636 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.56, "tpot_relative": 2.996349, "ttft_error_pct": 16.69, "ttft_relative": 3.168899 }, "aisimulate": { "tpot_error_pct": 29.1, "tpot_relative": 3.156253, "ttft_error_pct": 29.6, "ttft_relative": 2.677712 }, "concurrency": 128, "measured": { "tpot_relative": 2.444732, "ttft_relative": 3.803716 }, "status": "success" } ], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 10.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 87.9, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 20.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 67.44, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "d7149e4077fa52fa", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 10.97, "tpot_relative": 1.109678, "ttft_error_pct": 87.9, "ttft_relative": 0.120987 }, "aisimulate": { "tpot_error_pct": 20.56, "tpot_relative": 1.205611, "ttft_error_pct": 67.44, "ttft_relative": 0.325552 }, "concurrency": 512, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp8", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 22.88, "tpot_shape_error_pct": 1.66, "ttft_mape_pct": 62.8, "ttft_shape_error_pct": 48.12 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 32.82, "tpot_shape_error_pct": 1.82, "ttft_mape_pct": 33.56, "ttft_shape_error_pct": 10.55 }, "framework": "vllm", "id": "866510247643199b", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 23.91, "tpot_relative": 1.239111, "ttft_error_pct": 51.02, "ttft_relative": 0.489846 }, "aisimulate": { "tpot_error_pct": 34.04, "tpot_relative": 1.340414, "ttft_error_pct": 29.86, "ttft_relative": 0.701408 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.85, "tpot_relative": 1.703568, "ttft_error_pct": 74.59, "ttft_relative": 0.52126 }, "aisimulate": { "tpot_error_pct": 31.6, "tpot_relative": 1.839895, "ttft_error_pct": 37.26, "ttft_relative": 1.28687 }, "concurrency": 512, "measured": { "tpot_relative": 1.398096, "ttft_relative": 2.051031 }, "status": "success" } ], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 29, "tpot_mape_pct": 15.33, "tpot_shape_error_pct": 4.44, "ttft_mape_pct": 61.15, "ttft_shape_error_pct": 34.16 }, "aisimulate": { "coverage_pct": 100.0, "points": 29, "status_counts": { "failed": 0, "success": 29, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.88, "tpot_shape_error_pct": 3.22, "ttft_mape_pct": 48.02, "ttft_shape_error_pct": 9.59 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 29, "topologies": [ { "aic": { "points": 3, "tpot_mape_pct": 21.82, "tpot_shape_error_pct": 8.08, "ttft_mape_pct": 62.36, "ttft_shape_error_pct": 42.33 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.79, "tpot_shape_error_pct": 7.35, "ttft_mape_pct": 52.96, "ttft_shape_error_pct": 9.43 }, "framework": "dynamo-vllm", "id": "65243ceef43d6e22", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 15.6, "tpot_relative": 1.155984, "ttft_error_pct": 47.57, "ttft_relative": 0.524329 }, "aisimulate": { "tpot_error_pct": 6.56, "tpot_relative": 1.065612, "ttft_error_pct": 49.81, "ttft_relative": 0.501928 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.83, "tpot_relative": 1.425434, "ttft_error_pct": 64.21, "ttft_relative": 0.524329 }, "aisimulate": { "tpot_error_pct": 14.87, "tpot_relative": 1.311702, "ttft_error_pct": 54.1, "ttft_relative": 0.672515 }, "concurrency": 64, "measured": { "tpot_relative": 1.141917, "ttft_relative": 1.465202 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.04, "tpot_relative": 1.676371, "ttft_error_pct": 75.31, "ttft_relative": 0.524329 }, "aisimulate": { "tpot_error_pct": 13.93, "tpot_relative": 1.527328, "ttft_error_pct": 54.98, "ttft_relative": 0.956281 }, "concurrency": 128, "measured": { "tpot_relative": 1.340638, "ttft_relative": 2.123936 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 24.3, "tpot_shape_error_pct": 2.44, "ttft_mape_pct": 22.63, "ttft_shape_error_pct": 24.26 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.81, "tpot_shape_error_pct": 2.2, "ttft_mape_pct": 44.61, "ttft_shape_error_pct": 2.67 }, "framework": "dynamo-vllm", "id": "e569049ae9c99dec", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 22.8, "tpot_relative": 1.227962, "ttft_error_pct": 11.95, "ttft_relative": 0.880489 }, "aisimulate": { "tpot_error_pct": 13.57, "tpot_relative": 1.135654, "ttft_error_pct": 43.86, "ttft_relative": 0.561416 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.79, "tpot_relative": 1.521621, "ttft_error_pct": 33.31, "ttft_relative": 0.880489 }, "aisimulate": { "tpot_error_pct": 16.06, "tpot_relative": 1.403925, "ttft_error_pct": 45.36, "ttft_relative": 0.721468 }, "concurrency": 16, "measured": { "tpot_relative": 1.209614, "ttft_relative": 1.320345 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 18.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 95.13, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 23.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 731.55, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "22dcaeb591172b1c", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 18.08, "tpot_relative": 0.819236, "ttft_error_pct": 95.13, "ttft_relative": 0.048659 }, "aisimulate": { "tpot_error_pct": 23.39, "tpot_relative": 0.766052, "ttft_error_pct": 731.55, "ttft_relative": 8.315514 }, "concurrency": 2048, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 47.89, "tpot_shape_error_pct": 2.93, "ttft_mape_pct": 69.45, "ttft_shape_error_pct": 6.5 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 48.61, "tpot_shape_error_pct": 2.83, "ttft_mape_pct": 3.29, "ttft_shape_error_pct": 4.49 }, "framework": "trtllm", "id": "0053a0cd71b84514", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 50.09, "tpot_relative": 1.500861, "ttft_error_pct": 75.14, "ttft_relative": 1.751429 }, "aisimulate": { "tpot_error_pct": 50.74, "tpot_relative": 1.50744, "ttft_error_pct": 5.66, "ttft_relative": 1.056614 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 45.69, "tpot_relative": 1.800622, "ttft_error_pct": 63.76, "ttft_relative": 1.926555 }, "aisimulate": { "tpot_error_pct": 46.48, "tpot_relative": 1.810422, "ttft_error_pct": 0.92, "ttft_relative": 1.187253 }, "concurrency": 8, "measured": { "tpot_relative": 1.235932, "ttft_relative": 1.176447 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 26.16, "tpot_shape_error_pct": 4.48, "ttft_mape_pct": 10.76, "ttft_shape_error_pct": 8.54 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 26.83, "tpot_shape_error_pct": 4.32, "ttft_mape_pct": 30.81, "ttft_shape_error_pct": 7.43 }, "framework": "trtllm", "id": "8408bed5699bd4de", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 29.05, "tpot_relative": 1.290483, "ttft_error_pct": 15.7, "ttft_relative": 1.156983 }, "aisimulate": { "tpot_error_pct": 29.63, "tpot_relative": 1.296321, "ttft_error_pct": 28.14, "ttft_relative": 0.718639 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.27, "tpot_relative": 1.387247, "ttft_error_pct": 5.82, "ttft_relative": 1.271187 }, "aisimulate": { "tpot_error_pct": 24.04, "tpot_relative": 1.395904, "ttft_error_pct": 33.48, "ttft_relative": 0.799126 }, "concurrency": 8, "measured": { "tpot_relative": 1.125397, "ttft_relative": 1.201254 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 8, "tpot_mape_pct": 4.46, "tpot_shape_error_pct": 5.73, "ttft_mape_pct": 60.51, "ttft_shape_error_pct": 47.73 }, "aisimulate": { "points": 8, "status_counts": { "failed": 0, "success": 8, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 3.46, "tpot_shape_error_pct": 2.92, "ttft_mape_pct": 20.39, "ttft_shape_error_pct": 9.57 }, "framework": "vllm", "id": "346879997ea13503", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 2.71, "tpot_relative": 1.027129, "ttft_error_pct": 106.46, "ttft_relative": 2.064578 }, "aisimulate": { "tpot_error_pct": 2.31, "tpot_relative": 1.023067, "ttft_error_pct": 25.25, "ttft_relative": 0.747453 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.95, "tpot_relative": 1.173416, "ttft_error_pct": 97.06, "ttft_relative": 2.271801 }, "aisimulate": { "tpot_error_pct": 4.89, "tpot_relative": 1.172699, "ttft_error_pct": 25.95, "ttft_relative": 0.85376 }, "concurrency": 8, "measured": { "tpot_relative": 1.118024, "ttft_relative": 1.152875 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.68, "tpot_relative": 1.361987, "ttft_error_pct": 72.07, "ttft_relative": 2.48075 }, "aisimulate": { "tpot_error_pct": 3.05, "tpot_relative": 1.366849, "ttft_error_pct": 27.93, "ttft_relative": 1.039128 }, "concurrency": 16, "measured": { "tpot_relative": 1.32642, "ttft_relative": 1.441746 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.55, "tpot_relative": 1.574357, "ttft_error_pct": 40.64, "ttft_relative": 2.691332 }, "aisimulate": { "tpot_error_pct": 7.39, "tpot_relative": 1.594179, "ttft_error_pct": 27.46, "ttft_relative": 1.388146 }, "concurrency": 32, "measured": { "tpot_relative": 1.721469, "ttft_relative": 1.913688 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.34, "tpot_relative": 2.118386, "ttft_error_pct": 6.77, "ttft_relative": 2.928257 }, "aisimulate": { "tpot_error_pct": 1.69, "tpot_relative": 2.161744, "ttft_error_pct": 22.3, "ttft_relative": 2.130984 }, "concurrency": 64, "measured": { "tpot_relative": 2.125713, "ttft_relative": 2.742506 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.56, "tpot_relative": 2.573981, "ttft_error_pct": 30.19, "ttft_relative": 3.192877 }, "aisimulate": { "tpot_error_pct": 1.68, "tpot_relative": 2.686122, "ttft_error_pct": 20.47, "ttft_relative": 3.637457 }, "concurrency": 128, "measured": { "tpot_relative": 2.641694, "ttft_relative": 4.573563 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.53, "tpot_relative": 3.383741, "ttft_error_pct": 54.23, "ttft_relative": 3.537504 }, "aisimulate": { "tpot_error_pct": 0.24, "tpot_relative": 3.61126, "ttft_error_pct": 10.43, "ttft_relative": 6.922021 }, "concurrency": 256, "measured": { "tpot_relative": 3.620029, "ttft_relative": 7.728106 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.36, "tpot_relative": 4.821749, "ttft_error_pct": 76.64, "ttft_relative": 3.70066 }, "aisimulate": { "tpot_error_pct": 6.43, "tpot_relative": 5.539584, "ttft_error_pct": 3.34, "ttft_relative": 15.31489 }, "concurrency": 512, "measured": { "tpot_relative": 5.204826, "ttft_relative": 15.844147 }, "status": "success" } ], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 19.02, "tpot_shape_error_pct": null, "ttft_mape_pct": 119.88, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.98, "tpot_shape_error_pct": null, "ttft_mape_pct": 28.28, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "11d1688e16b99550", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 19.02, "tpot_relative": 1.190165, "ttft_error_pct": 119.88, "ttft_relative": 2.198757 }, "aisimulate": { "tpot_error_pct": 18.98, "tpot_relative": 1.189808, "ttft_error_pct": 28.28, "ttft_relative": 0.71719 }, "concurrency": 8, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 9.31, "tpot_shape_error_pct": null, "ttft_mape_pct": 10.96, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.6, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.96, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "e630b8582cafcee1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 9.31, "tpot_relative": 1.093095, "ttft_error_pct": 10.96, "ttft_relative": 0.89044 }, "aisimulate": { "tpot_error_pct": 14.6, "tpot_relative": 1.146032, "ttft_error_pct": 0.96, "ttft_relative": 0.990417 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 3.24, "tpot_shape_error_pct": 3.14, "ttft_mape_pct": 54.47, "ttft_shape_error_pct": 49.56 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.75, "tpot_shape_error_pct": 4.26, "ttft_mape_pct": 7.68, "ttft_shape_error_pct": 14.13 }, "framework": "vllm", "id": "ceeecd5e7c74cb83", "parallelism": { "attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 5.22, "tpot_relative": 0.947769, "ttft_error_pct": 32.0, "ttft_relative": 0.67999 }, "aisimulate": { "tpot_error_pct": 7.7, "tpot_relative": 1.076963, "ttft_error_pct": 15.17, "ttft_relative": 0.84832 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.79, "tpot_relative": 1.274307, "ttft_error_pct": 55.56, "ttft_relative": 0.773831 }, "aisimulate": { "tpot_error_pct": 12.54, "tpot_relative": 1.475264, "ttft_error_pct": 7.11, "ttft_relative": 1.617581 }, "concurrency": 512, "measured": { "tpot_relative": 1.310883, "ttft_relative": 1.741447 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.7, "tpot_relative": 1.887848, "ttft_error_pct": 75.84, "ttft_relative": 0.859056 }, "aisimulate": { "tpot_error_pct": 12.03, "tpot_relative": 2.151464, "ttft_error_pct": 0.76, "ttft_relative": 3.582345 }, "concurrency": 1024, "measured": { "tpot_relative": 1.920469, "ttft_relative": 3.555379 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 13.44, "tpot_shape_error_pct": 0.87, "ttft_mape_pct": 94.49, "ttft_shape_error_pct": 1.45 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.85, "tpot_shape_error_pct": 0.72, "ttft_mape_pct": 19.53, "ttft_shape_error_pct": 6.4 }, "framework": "vllm", "id": "e4236d087f29a5f1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 13.94, "tpot_relative": 1.139369, "ttft_error_pct": 93.09, "ttft_relative": 1.930932 }, "aisimulate": { "tpot_error_pct": 13.26, "tpot_relative": 1.132599, "ttft_error_pct": 22.02, "ttft_relative": 0.779776 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.95, "tpot_relative": 1.419219, "ttft_error_pct": 95.89, "ttft_relative": 2.12536 }, "aisimulate": { "tpot_error_pct": 12.44, "tpot_relative": 1.412888, "ttft_error_pct": 17.03, "ttft_relative": 0.900194 }, "concurrency": 8, "measured": { "tpot_relative": 1.256537, "ttft_relative": 1.084975 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 10.44, "tpot_shape_error_pct": 1.45, "ttft_mape_pct": 11.62, "ttft_shape_error_pct": 19.37 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.14, "tpot_shape_error_pct": 0.13, "ttft_mape_pct": 17.59, "ttft_shape_error_pct": 27.13 }, "framework": "vllm", "id": "d21ae151bccfef65", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 11.25, "tpot_relative": 1.112468, "ttft_error_pct": 19.96, "ttft_relative": 1.199554 }, "aisimulate": { "tpot_error_pct": 15.06, "tpot_relative": 1.150629, "ttft_error_pct": 27.43, "ttft_relative": 0.725663 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.64, "tpot_relative": 1.351773, "ttft_error_pct": 3.28, "ttft_relative": 1.303639 }, "aisimulate": { "tpot_error_pct": 15.22, "tpot_relative": 1.420543, "ttft_error_pct": 7.75, "ttft_relative": 1.243364 }, "concurrency": 128, "measured": { "tpot_relative": 1.232931, "ttft_relative": 1.347783 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 21.42, "tpot_shape_error_pct": 0.74, "ttft_mape_pct": 147.41, "ttft_shape_error_pct": 2.74 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 21.35, "tpot_shape_error_pct": 0.9, "ttft_mape_pct": 27.61, "ttft_shape_error_pct": 0.82 }, "framework": "vllm", "id": "1b3c46329839b145", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 20.97, "tpot_relative": 1.209705, "ttft_error_pct": 150.85, "ttft_relative": 2.508472 }, "aisimulate": { "tpot_error_pct": 20.8, "tpot_relative": 1.208015, "ttft_error_pct": 27.9, "ttft_relative": 0.720979 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.87, "tpot_relative": 1.334142, "ttft_error_pct": 143.97, "ttft_relative": 2.761271 }, "aisimulate": { "tpot_error_pct": 21.89, "tpot_relative": 1.334396, "ttft_error_pct": 27.31, "ttft_relative": 0.822676 }, "concurrency": 8, "measured": { "tpot_relative": 1.094743, "ttft_relative": 1.131829 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 15, "tpot_mape_pct": 10.99, "tpot_shape_error_pct": 10.49, "ttft_mape_pct": 17.75, "ttft_shape_error_pct": 24.21 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.91, "tpot_shape_error_pct": 10.67, "ttft_mape_pct": 49.27, "ttft_shape_error_pct": 30.07 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 15, "topologies": [ { "aic": { "points": 9, "tpot_mape_pct": 12.94, "tpot_shape_error_pct": 14.39, "ttft_mape_pct": 19.56, "ttft_shape_error_pct": 19.28 }, "aisimulate": { "points": 9, "status_counts": { "failed": 0, "success": 9, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.94, "tpot_shape_error_pct": 15.13, "ttft_mape_pct": 47.72, "ttft_shape_error_pct": 35.09 }, "framework": "vllm", "id": "96c490db5243a8bc", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 13.82, "tpot_relative": 0.861814, "ttft_error_pct": 15.32, "ttft_relative": 0.846782 }, "aisimulate": { "tpot_error_pct": 13.87, "tpot_relative": 0.861258, "ttft_error_pct": 57.95, "ttft_relative": 0.420542 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.45, "tpot_relative": 0.974169, "ttft_error_pct": 26.17, "ttft_relative": 0.987829 }, "aisimulate": { "tpot_error_pct": 9.09, "tpot_relative": 0.967383, "ttft_error_pct": 67.22, "ttft_relative": 0.438668 }, "concurrency": 2, "measured": { "tpot_relative": 1.064126, "ttft_relative": 1.338039 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.76, "tpot_relative": 1.098639, "ttft_error_pct": 10.66, "ttft_relative": 1.098272 }, "aisimulate": { "tpot_error_pct": 5.45, "tpot_relative": 1.090685, "ttft_error_pct": 61.7, "ttft_relative": 0.470876 }, "concurrency": 4, "measured": { "tpot_relative": 1.153549, "ttft_relative": 1.229382 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.65, "tpot_relative": 1.319644, "ttft_error_pct": 1.95, "ttft_relative": 1.209427 }, "aisimulate": { "tpot_error_pct": 4.07, "tpot_relative": 1.313861, "ttft_error_pct": 54.73, "ttft_relative": 0.537081 }, "concurrency": 8, "measured": { "tpot_relative": 1.369595, "ttft_relative": 1.186345 }, "status": "success" }, { "aic": { "tpot_error_pct": 47.86, "tpot_relative": 1.660444, "ttft_error_pct": 20.64, "ttft_relative": 1.322685 }, "aisimulate": { "tpot_error_pct": 47.86, "tpot_relative": 1.660598, "ttft_error_pct": 43.02, "ttft_relative": 0.624768 }, "concurrency": 16, "measured": { "tpot_relative": 3.184798, "ttft_relative": 1.096406 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.77, "tpot_relative": 2.023523, "ttft_error_pct": 10.43, "ttft_relative": 1.44026 }, "aisimulate": { "tpot_error_pct": 10.16, "tpot_relative": 2.037253, "ttft_error_pct": 51.31, "ttft_relative": 0.782957 }, "concurrency": 32, "measured": { "tpot_relative": 2.267701, "ttft_relative": 1.607967 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.73, "tpot_relative": 2.852565, "ttft_error_pct": 25.38, "ttft_relative": 1.56494 }, "aisimulate": { "tpot_error_pct": 0.49, "tpot_relative": 2.916792, "ttft_error_pct": 47.56, "ttft_relative": 1.099907 }, "concurrency": 64, "measured": { "tpot_relative": 2.902703, "ttft_relative": 2.097322 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.56, "tpot_relative": 3.473838, "ttft_error_pct": 9.27, "ttft_relative": 1.715709 }, "aisimulate": { "tpot_error_pct": 13.43, "tpot_relative": 3.629875, "ttft_error_pct": 11.32, "ttft_relative": 1.676996 }, "concurrency": 128, "measured": { "tpot_relative": 3.199982, "ttft_relative": 1.890978 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.84, "tpot_relative": 4.461991, "ttft_error_pct": 56.19, "ttft_relative": 1.906623 }, "aisimulate": { "tpot_error_pct": 12.02, "tpot_relative": 4.72082, "ttft_error_pct": 34.7, "ttft_relative": 2.84145 }, "concurrency": 256, "measured": { "tpot_relative": 5.365777, "ttft_relative": 4.351627 }, "status": "success" } ], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 8.08, "tpot_shape_error_pct": 4.26, "ttft_mape_pct": 15.04, "ttft_shape_error_pct": 32.1 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.86, "tpot_shape_error_pct": 3.55, "ttft_mape_pct": 51.59, "ttft_shape_error_pct": 22.03 }, "framework": "vllm", "id": "e070f07b028568e6", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 7.17, "tpot_relative": 0.92832, "ttft_error_pct": 29.34, "ttft_relative": 1.293377 }, "aisimulate": { "tpot_error_pct": 7.69, "tpot_relative": 0.923094, "ttft_error_pct": 50.34, "ttft_relative": 0.496566 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.75, "tpot_relative": 1.066109, "ttft_error_pct": 2.42, "ttft_relative": 1.423861 }, "aisimulate": { "tpot_error_pct": 9.12, "tpot_relative": 1.061789, "ttft_error_pct": 61.23, "ttft_relative": 0.56573 }, "concurrency": 8, "measured": { "tpot_relative": 1.168386, "ttft_relative": 1.459246 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.93, "tpot_relative": 1.27843, "ttft_error_pct": 8.83, "ttft_relative": 1.556513 }, "aisimulate": { "tpot_error_pct": 8.81, "tpot_relative": 1.280116, "ttft_error_pct": 61.59, "ttft_relative": 0.655682 }, "concurrency": 16, "measured": { "tpot_relative": 1.403721, "ttft_relative": 1.707243 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.94, "tpot_relative": 1.561279, "ttft_error_pct": 16.8, "ttft_relative": 1.691806 }, "aisimulate": { "tpot_error_pct": 10.18, "tpot_relative": 1.574671, "ttft_error_pct": 59.29, "ttft_relative": 0.827744 }, "concurrency": 32, "measured": { "tpot_relative": 1.7531, "ttft_relative": 2.0333 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.41, "tpot_relative": 2.094859, "ttft_error_pct": 1.27, "ttft_relative": 1.836215 }, "aisimulate": { "tpot_error_pct": 2.82, "tpot_relative": 2.144988, "ttft_error_pct": 37.6, "ttft_relative": 1.160648 }, "concurrency": 64, "measured": { "tpot_relative": 2.086233, "ttft_relative": 1.85989 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.27, "tpot_relative": 2.384155, "ttft_error_pct": 31.57, "ttft_relative": 1.993618 }, "aisimulate": { "tpot_error_pct": 8.53, "tpot_relative": 2.485621, "ttft_error_pct": 39.48, "ttft_relative": 1.762979 }, "concurrency": 128, "measured": { "tpot_relative": 2.717468, "ttft_relative": 2.913248 }, "status": "success" } ], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4", "fp8" ], "rows": 80 }, { "aic": { "points": 88, "tpot_mape_pct": 15.62, "tpot_shape_error_pct": 6.53, "ttft_mape_pct": 43.3, "ttft_shape_error_pct": 40.59 }, "aisimulate": { "coverage_pct": 100.0, "points": 88, "status_counts": { "failed": 0, "success": 88, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.69, "tpot_shape_error_pct": 5.66, "ttft_mape_pct": 37.09, "ttft_shape_error_pct": 13.84 }, "gpu_skus": [ "b200", "b300", "h200" ], "gpus": [ { "aic": { "points": 38, "tpot_mape_pct": 17.84, "tpot_shape_error_pct": 8.51, "ttft_mape_pct": 42.01, "ttft_shape_error_pct": 39.95 }, "aisimulate": { "coverage_pct": 100.0, "points": 38, "status_counts": { "failed": 0, "success": 38, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.78, "tpot_shape_error_pct": 6.57, "ttft_mape_pct": 33.9, "ttft_shape_error_pct": 13.91 }, "gpu": "b200", "precisions": [ "fp4", "fp8" ], "rows": 38, "topologies": [ { "aic": { "points": 3, "tpot_mape_pct": 20.19, "tpot_shape_error_pct": 0.58, "ttft_mape_pct": 34.26, "ttft_shape_error_pct": 25.64 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.24, "tpot_shape_error_pct": 0.4, "ttft_mape_pct": 48.38, "ttft_shape_error_pct": 15.39 }, "framework": "dynamo-vllm", "id": "117f7e9bb6f4e594", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 19.73, "tpot_relative": 1.197286, "ttft_error_pct": 20.7, "ttft_relative": 0.792977 }, "aisimulate": { "tpot_error_pct": 9.95, "tpot_relative": 1.099502, "ttft_error_pct": 53.18, "ttft_relative": 0.468221 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.42, "tpot_relative": 1.319316, "ttft_error_pct": 34.67, "ttft_relative": 0.792977 }, "aisimulate": { "tpot_error_pct": 10.78, "tpot_relative": 1.21373, "ttft_error_pct": 45.94, "ttft_relative": 0.656265 }, "concurrency": 8, "measured": { "tpot_relative": 1.09559, "ttft_relative": 1.213852 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.43, "tpot_relative": 1.46105, "ttft_error_pct": 47.4, "ttft_relative": 0.792977 }, "aisimulate": { "tpot_error_pct": 9.99, "tpot_relative": 1.334396, "ttft_error_pct": 46.01, "ttft_relative": 0.813835 }, "concurrency": 16, "measured": { "tpot_relative": 1.213228, "ttft_relative": 1.507453 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 44.34, "tpot_shape_error_pct": 25.11, "ttft_mape_pct": 78.84, "ttft_shape_error_pct": 66.85 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 16.19, "tpot_shape_error_pct": 6.86, "ttft_mape_pct": 41.81, "ttft_shape_error_pct": 7.61 }, "framework": "dynamo-vllm", "id": "93b5a35bb26dc6d9", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 23.64, "tpot_relative": 1.236397, "ttft_error_pct": 61.82, "ttft_relative": 0.38182 }, "aisimulate": { "tpot_error_pct": 11.11, "tpot_relative": 1.111056, "ttft_error_pct": 41.86, "ttft_relative": 0.581395 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 29.93, "tpot_relative": 1.597643, "ttft_error_pct": 75.88, "ttft_relative": 0.38182 }, "aisimulate": { "tpot_error_pct": 15.78, "tpot_relative": 1.423725, "ttft_error_pct": 46.21, "ttft_relative": 0.851668 }, "concurrency": 64, "measured": { "tpot_relative": 1.229647, "ttft_relative": 1.583277 }, "status": "success" }, { "aic": { "tpot_error_pct": 79.44, "tpot_relative": 2.784896, "ttft_error_pct": 98.8, "ttft_relative": 0.38182 }, "aisimulate": { "tpot_error_pct": 21.67, "tpot_relative": 1.888293, "ttft_error_pct": 37.36, "ttft_relative": 19.994039 }, "concurrency": 256, "measured": { "tpot_relative": 1.551987, "ttft_relative": 31.919236 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 16.12, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.07, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 0.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 40.49, "ttft_shape_error_pct": null }, "framework": "dynamo-vllm", "id": "ccff69f1c62d408a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 16.12, "tpot_relative": 1.161206, "ttft_error_pct": 84.07, "ttft_relative": 0.159256 }, "aisimulate": { "tpot_error_pct": 0.5, "tpot_relative": 1.005037, "ttft_error_pct": 40.49, "ttft_relative": 0.595097 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 30.53, "tpot_shape_error_pct": 17.27, "ttft_mape_pct": 71.55, "ttft_shape_error_pct": 60.74 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.06, "tpot_shape_error_pct": 9.66, "ttft_mape_pct": 35.0, "ttft_shape_error_pct": 19.5 }, "framework": "dynamo-vllm", "id": "12d2bc1e543712fc", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 17.64, "tpot_relative": 1.17643, "ttft_error_pct": 44.65, "ttft_relative": 0.553504 }, "aisimulate": { "tpot_error_pct": 7.87, "tpot_relative": 1.07869, "ttft_error_pct": 43.78, "ttft_relative": 0.562245 }, "concurrency": 16, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.24, "tpot_relative": 1.406712, "ttft_error_pct": 59.16, "ttft_relative": 0.553504 }, "aisimulate": { "tpot_error_pct": 1.35, "tpot_relative": 1.270421, "ttft_error_pct": 37.63, "ttft_relative": 0.84527 }, "concurrency": 32, "measured": { "tpot_relative": 1.287782, "ttft_relative": 1.355348 }, "status": "success" }, { "aic": { "tpot_error_pct": 29.93, "tpot_relative": 2.044084, "ttft_error_pct": 71.64, "ttft_relative": 0.553504 }, "aisimulate": { "tpot_error_pct": 16.52, "tpot_relative": 1.833049, "ttft_error_pct": 38.18, "ttft_relative": 1.206379 }, "concurrency": 64, "measured": { "tpot_relative": 1.573213, "ttft_relative": 1.951367 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.67, "tpot_relative": 2.578531, "ttft_error_pct": 84.21, "ttft_relative": 0.553504 }, "aisimulate": { "tpot_error_pct": 15.73, "tpot_relative": 2.283795, "ttft_error_pct": 32.78, "ttft_relative": 2.356383 }, "concurrency": 128, "measured": { "tpot_relative": 1.973389, "ttft_relative": 3.50568 }, "status": "success" }, { "aic": { "tpot_error_pct": 65.2, "tpot_relative": 3.426365, "ttft_error_pct": 98.07, "ttft_relative": 0.553504 }, "aisimulate": { "tpot_error_pct": 23.82, "tpot_relative": 2.568195, "ttft_error_pct": 22.65, "ttft_relative": 22.194107 }, "concurrency": 256, "measured": { "tpot_relative": 2.0741, "ttft_relative": 28.693295 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 7.04, "tpot_shape_error_pct": 4.02, "ttft_mape_pct": 37.47, "ttft_shape_error_pct": 11.8 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.98, "tpot_shape_error_pct": 5.32, "ttft_mape_pct": 16.26, "ttft_shape_error_pct": 7.64 }, "framework": "trtllm", "id": "b7c874a507445779", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 8.92, "tpot_relative": 1.089203, "ttft_error_pct": 50.81, "ttft_relative": 1.508145 }, "aisimulate": { "tpot_error_pct": 11.46, "tpot_relative": 1.114567, "ttft_error_pct": 20.8, "ttft_relative": 0.791984 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.74, "tpot_relative": 1.457553, "ttft_error_pct": 44.99, "ttft_relative": 1.664032 }, "aisimulate": { "tpot_error_pct": 12.5, "tpot_relative": 1.467555, "ttft_error_pct": 14.62, "ttft_relative": 0.979901 }, "concurrency": 8, "measured": { "tpot_relative": 1.304452, "ttft_relative": 1.147688 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.07, "tpot_relative": 1.988318, "ttft_error_pct": 30.1, "ttft_relative": 1.975107 }, "aisimulate": { "tpot_error_pct": 7.07, "tpot_relative": 1.988271, "ttft_error_pct": 17.17, "ttft_relative": 1.257547 }, "concurrency": 16, "measured": { "tpot_relative": 1.857024, "ttft_relative": 1.518144 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.45, "tpot_relative": 2.826417, "ttft_error_pct": 23.98, "ttft_relative": 2.620037 }, "aisimulate": { "tpot_error_pct": 0.89, "tpot_relative": 2.788679, "ttft_error_pct": 12.47, "ttft_relative": 1.849764 }, "concurrency": 32, "measured": { "tpot_relative": 2.813798, "ttft_relative": 2.11332 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 3.58, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.24, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 1.46, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.8, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "e78ba112d4f578d1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 3.58, "tpot_relative": 0.964216, "ttft_error_pct": 16.24, "ttft_relative": 1.162355 }, "aisimulate": { "tpot_error_pct": 1.46, "tpot_relative": 0.985403, "ttft_error_pct": 36.8, "ttft_relative": 0.631953 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 3.76, "tpot_shape_error_pct": null, "ttft_mape_pct": 92.59, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 57.38, "tpot_shape_error_pct": null, "ttft_mape_pct": 92.69, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "66813521aa593977", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 3.76, "tpot_relative": 0.962385, "ttft_error_pct": 92.59, "ttft_relative": 0.074113 }, "aisimulate": { "tpot_error_pct": 57.38, "tpot_relative": 1.573776, "ttft_error_pct": 92.69, "ttft_relative": 0.073096 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 11.69, "tpot_shape_error_pct": 5.02, "ttft_mape_pct": 28.39, "ttft_shape_error_pct": 38.59 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.66, "tpot_shape_error_pct": 5.04, "ttft_mape_pct": 46.91, "ttft_shape_error_pct": 23.26 }, "framework": "vllm", "id": "c53059f4783dcb95", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 11.51, "tpot_relative": 1.115132, "ttft_error_pct": 2.62, "ttft_relative": 1.026151 }, "aisimulate": { "tpot_error_pct": 8.64, "tpot_relative": 1.086391, "ttft_error_pct": 35.69, "ttft_relative": 0.643094 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.04, "tpot_relative": 1.542786, "ttft_error_pct": 11.28, "ttft_relative": 1.134046 }, "aisimulate": { "tpot_error_pct": 14.73, "tpot_relative": 1.499618, "ttft_error_pct": 39.03, "ttft_relative": 0.779339 }, "concurrency": 8, "measured": { "tpot_relative": 1.307049, "ttft_relative": 1.278243 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.73, "tpot_relative": 2.091354, "ttft_error_pct": 23.69, "ttft_relative": 1.246512 }, "aisimulate": { "tpot_error_pct": 10.81, "tpot_relative": 2.037553, "ttft_error_pct": 37.71, "ttft_relative": 1.01739 }, "concurrency": 16, "measured": { "tpot_relative": 1.838808, "ttft_relative": 1.63342 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.46, "tpot_relative": 2.943067, "ttft_error_pct": 75.99, "ttft_relative": 1.374117 }, "aisimulate": { "tpot_error_pct": 0.46, "tpot_relative": 2.857683, "ttft_error_pct": 75.2, "ttft_relative": 1.419198 }, "concurrency": 32, "measured": { "tpot_relative": 2.844526, "ttft_relative": 5.723665 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 17.79, "tpot_shape_error_pct": 0.03, "ttft_mape_pct": 5.36, "ttft_shape_error_pct": 9.9 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 16.18, "tpot_shape_error_pct": 0.23, "ttft_mape_pct": 42.67, "ttft_shape_error_pct": 2.9 }, "framework": "vllm", "id": "20f961a65bec681d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 17.81, "tpot_relative": 1.178086, "ttft_error_pct": 8.22, "ttft_relative": 1.082173 }, "aisimulate": { "tpot_error_pct": 16.05, "tpot_relative": 1.160479, "ttft_error_pct": 41.82, "ttft_relative": 0.581756 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.77, "tpot_relative": 1.33738, "ttft_error_pct": 2.5, "ttft_relative": 1.192062 }, "aisimulate": { "tpot_error_pct": 16.31, "tpot_relative": 1.32082, "ttft_error_pct": 43.51, "ttft_relative": 0.690639 }, "concurrency": 8, "measured": { "tpot_relative": 1.135574, "ttft_relative": 1.222641 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 20.74, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.64, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 19.45, "tpot_shape_error_pct": null, "ttft_mape_pct": 48.35, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "45158c00823244e8", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 20.74, "tpot_relative": 1.207366, "ttft_error_pct": 0.64, "ttft_relative": 1.00636 }, "aisimulate": { "tpot_error_pct": 19.45, "tpot_relative": 1.194526, "ttft_error_pct": 48.35, "ttft_relative": 0.516471 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 4.79, "tpot_shape_error_pct": null, "ttft_mape_pct": 64.18, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 0.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 25.02, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "bcbf78706b68a7cc", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 4.79, "tpot_relative": 1.04789, "ttft_error_pct": 64.18, "ttft_relative": 0.35821 }, "aisimulate": { "tpot_error_pct": 0.97, "tpot_relative": 1.009737, "ttft_error_pct": 25.02, "ttft_relative": 0.749783 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 14.0, "tpot_shape_error_pct": 3.03, "ttft_mape_pct": 23.05, "ttft_shape_error_pct": 31.53 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.15, "tpot_shape_error_pct": 3.34, "ttft_mape_pct": 26.14, "ttft_shape_error_pct": 6.45 }, "framework": "vllm", "id": "7b351e6c6b5d71a2", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 12.71, "tpot_relative": 1.127117, "ttft_error_pct": 6.23, "ttft_relative": 1.06231 }, "aisimulate": { "tpot_error_pct": 10.37, "tpot_relative": 1.103651, "ttft_error_pct": 29.76, "ttft_relative": 0.702355 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.37, "tpot_relative": 1.378016, "ttft_error_pct": 3.36, "ttft_relative": 1.169552 }, "aisimulate": { "tpot_error_pct": 10.92, "tpot_relative": 1.34824, "ttft_error_pct": 28.61, "ttft_relative": 0.863969 }, "concurrency": 8, "measured": { "tpot_relative": 1.215543, "ttft_relative": 1.210243 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.43, "tpot_relative": 1.863398, "ttft_error_pct": 17.13, "ttft_relative": 1.282025 }, "aisimulate": { "tpot_error_pct": 11.18, "tpot_relative": 1.826407, "ttft_error_pct": 28.82, "ttft_relative": 1.101259 }, "concurrency": 16, "measured": { "tpot_relative": 1.642742, "ttft_relative": 1.547039 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.11, "tpot_relative": 2.49017, "ttft_error_pct": 35.48, "ttft_relative": 1.398726 }, "aisimulate": { "tpot_error_pct": 7.47, "tpot_relative": 2.452739, "ttft_error_pct": 25.66, "ttft_relative": 1.611567 }, "concurrency": 32, "measured": { "tpot_relative": 2.282316, "ttft_relative": 2.167977 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.41, "tpot_relative": 4.086159, "ttft_error_pct": 53.06, "ttft_relative": 1.531308 }, "aisimulate": { "tpot_error_pct": 20.83, "tpot_relative": 4.06661, "ttft_error_pct": 17.86, "ttft_relative": 2.679896 }, "concurrency": 64, "measured": { "tpot_relative": 3.365675, "ttft_relative": 3.262416 }, "status": "success" } ], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 14.58, "tpot_shape_error_pct": 8.81, "ttft_mape_pct": 36.01, "ttft_shape_error_pct": 47.25 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.53, "tpot_shape_error_pct": 11.06, "ttft_mape_pct": 18.63, "ttft_shape_error_pct": 17.07 }, "framework": "vllm", "id": "c584ece0758a3d1c", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 6.53, "tpot_relative": 1.065314, "ttft_error_pct": 19.49, "ttft_relative": 1.194926 }, "aisimulate": { "tpot_error_pct": 4.61, "tpot_relative": 1.046127, "ttft_error_pct": 25.51, "ttft_relative": 0.744906 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.83, "tpot_relative": 1.293203, "ttft_error_pct": 5.38, "ttft_relative": 1.315948 }, "aisimulate": { "tpot_error_pct": 10.3, "tpot_relative": 1.275464, "ttft_error_pct": 26.45, "ttft_relative": 0.918465 }, "concurrency": 8, "measured": { "tpot_relative": 1.156411, "ttft_relative": 1.248791 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.64, "tpot_relative": 1.625877, "ttft_error_pct": 7.6, "ttft_relative": 1.436856 }, "aisimulate": { "tpot_error_pct": 12.45, "tpot_relative": 1.608792, "ttft_error_pct": 25.21, "ttft_relative": 1.163061 }, "concurrency": 16, "measured": { "tpot_relative": 1.430696, "ttft_relative": 1.55502 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.8, "tpot_relative": 2.171006, "ttft_error_pct": 30.11, "ttft_relative": 1.567036 }, "aisimulate": { "tpot_error_pct": 11.85, "tpot_relative": 2.152889, "ttft_error_pct": 24.08, "ttft_relative": 1.702226 }, "concurrency": 32, "measured": { "tpot_relative": 1.924732, "ttft_relative": 2.24209 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.43, "tpot_relative": 3.383546, "ttft_error_pct": 49.2, "ttft_relative": 1.703314 }, "aisimulate": { "tpot_error_pct": 25.36, "tpot_relative": 3.408896, "ttft_error_pct": 16.03, "ttft_relative": 2.815674 }, "concurrency": 64, "measured": { "tpot_relative": 2.71928, "ttft_relative": 3.353026 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.6, "tpot_relative": 4.676007, "ttft_error_pct": 63.0, "ttft_relative": 1.865327 }, "aisimulate": { "tpot_error_pct": 20.24, "tpot_relative": 4.740392, "ttft_error_pct": 2.06, "ttft_relative": 5.14577 }, "concurrency": 128, "measured": { "tpot_relative": 3.942507, "ttft_relative": 5.041725 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.22, "tpot_relative": 7.208932, "ttft_error_pct": 77.28, "ttft_relative": 2.066552 }, "aisimulate": { "tpot_error_pct": 16.89, "tpot_relative": 7.377587, "ttft_error_pct": 11.05, "ttft_relative": 10.101484 }, "concurrency": 256, "measured": { "tpot_relative": 6.31132, "ttft_relative": 9.096193 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 35, "tpot_mape_pct": 16.74, "tpot_shape_error_pct": 5.21, "ttft_mape_pct": 47.21, "ttft_shape_error_pct": 43.75 }, "aisimulate": { "coverage_pct": 100.0, "points": 35, "status_counts": { "failed": 0, "success": 35, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.77, "tpot_shape_error_pct": 5.44, "ttft_mape_pct": 41.89, "ttft_shape_error_pct": 13.23 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 35, "topologies": [ { "aic": { "points": 2, "tpot_mape_pct": 22.29, "tpot_shape_error_pct": 1.31, "ttft_mape_pct": 79.55, "ttft_shape_error_pct": 19.82 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.59, "tpot_shape_error_pct": 0.51, "ttft_mape_pct": 35.46, "ttft_shape_error_pct": 0.91 }, "framework": "dynamo-vllm", "id": "d6ae1c91b617f2f4", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 21.5, "tpot_relative": 1.215003, "ttft_error_pct": 77.3, "ttft_relative": 0.227023 }, "aisimulate": { "tpot_error_pct": 4.33, "tpot_relative": 1.043274, "ttft_error_pct": 35.75, "ttft_relative": 0.642534 }, "concurrency": 384, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.09, "tpot_relative": 1.449384, "ttft_error_pct": 81.8, "ttft_relative": 0.227023 }, "aisimulate": { "tpot_error_pct": 4.86, "tpot_relative": 1.234704, "ttft_error_pct": 35.16, "ttft_relative": 0.808675 }, "concurrency": 512, "measured": { "tpot_relative": 1.17753, "ttft_relative": 1.247254 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 24.85, "tpot_shape_error_pct": 12.0, "ttft_mape_pct": 76.56, "ttft_shape_error_pct": 84.22 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 11.26, "tpot_shape_error_pct": 9.66, "ttft_mape_pct": 56.03, "ttft_shape_error_pct": 56.53 }, "framework": "dynamo-vllm", "id": "6219e9cc28f6d848", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 17.79, "tpot_relative": 1.177863, "ttft_error_pct": 59.5, "ttft_relative": 0.404959 }, "aisimulate": { "tpot_error_pct": 6.14, "tpot_relative": 1.061366, "ttft_error_pct": 38.7, "ttft_relative": 0.613005 }, "concurrency": 32, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 31.92, "tpot_relative": 2.01743, "ttft_error_pct": 93.61, "ttft_relative": 0.404959 }, "aisimulate": { "tpot_error_pct": 16.39, "tpot_relative": 1.779961, "ttft_error_pct": 73.35, "ttft_relative": 1.689092 }, "concurrency": 128, "measured": { "tpot_relative": 1.529316, "ttft_relative": 6.338954 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 18.49, "tpot_shape_error_pct": 5.22, "ttft_mape_pct": 22.4, "ttft_shape_error_pct": 27.7 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.84, "tpot_shape_error_pct": 4.57, "ttft_mape_pct": 46.04, "ttft_shape_error_pct": 7.87 }, "framework": "dynamo-vllm", "id": "331b6dacec6a0cf5", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 14.02, "tpot_relative": 1.140204, "ttft_error_pct": 2.05, "ttft_relative": 0.979529 }, "aisimulate": { "tpot_error_pct": 5.23, "tpot_relative": 1.052328, "ttft_error_pct": 49.05, "ttft_relative": 0.509501 }, "concurrency": 2, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.21, "tpot_relative": 1.230633, "ttft_error_pct": 12.64, "ttft_relative": 0.979529 }, "aisimulate": { "tpot_error_pct": 7.82, "tpot_relative": 1.132093, "ttft_error_pct": 47.29, "ttft_relative": 0.591007 }, "concurrency": 4, "measured": { "tpot_relative": 1.049983, "ttft_relative": 1.121287 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.42, "tpot_relative": 1.386859, "ttft_error_pct": 31.52, "ttft_relative": 0.979529 }, "aisimulate": { "tpot_error_pct": 11.59, "tpot_relative": 1.274616, "ttft_error_pct": 45.46, "ttft_relative": 0.780097 }, "concurrency": 8, "measured": { "tpot_relative": 1.142199, "ttft_relative": 1.430376 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.31, "tpot_relative": 1.547835, "ttft_error_pct": 43.38, "ttft_relative": 0.979529 }, "aisimulate": { "tpot_error_pct": 10.71, "tpot_relative": 1.412601, "ttft_error_pct": 42.37, "ttft_relative": 0.997052 }, "concurrency": 16, "measured": { "tpot_relative": 1.27596, "ttft_relative": 1.73 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 24.58, "tpot_shape_error_pct": 1.64, "ttft_mape_pct": 76.42, "ttft_shape_error_pct": 60.29 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.72, "tpot_shape_error_pct": 5.28, "ttft_mape_pct": 44.55, "ttft_shape_error_pct": 12.46 }, "framework": "dynamo-vllm", "id": "6cbb3e9b09e27f37", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 26.13, "tpot_relative": 1.261275, "ttft_error_pct": 56.95, "ttft_relative": 0.430451 }, "aisimulate": { "tpot_error_pct": 13.2, "tpot_relative": 1.131982, "ttft_error_pct": 43.62, "ttft_relative": 0.563814 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.48, "tpot_relative": 1.640599, "ttft_error_pct": 70.45, "ttft_relative": 0.430451 }, "aisimulate": { "tpot_error_pct": 11.45, "tpot_relative": 1.457093, "ttft_error_pct": 36.77, "ttft_relative": 0.920973 }, "concurrency": 128, "measured": { "tpot_relative": 1.307448, "ttft_relative": 1.456653 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.12, "tpot_relative": 2.302176, "ttft_error_pct": 84.11, "ttft_relative": 0.430451 }, "aisimulate": { "tpot_error_pct": 6.38, "tpot_relative": 1.98912, "ttft_error_pct": 41.78, "ttft_relative": 1.577659 }, "concurrency": 256, "measured": { "tpot_relative": 1.869846, "ttft_relative": 2.70966 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.58, "tpot_relative": 3.547348, "ttft_error_pct": 94.16, "ttft_relative": 0.430451 }, "aisimulate": { "tpot_error_pct": 3.85, "tpot_relative": 2.981072, "ttft_error_pct": 56.02, "ttft_relative": 3.24319 }, "concurrency": 512, "measured": { "tpot_relative": 2.870552, "ttft_relative": 7.373645 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 21.35, "tpot_shape_error_pct": 13.3, "ttft_mape_pct": 77.0, "ttft_shape_error_pct": 63.52 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 21.84, "tpot_shape_error_pct": 11.24, "ttft_mape_pct": 80.14, "ttft_shape_error_pct": 46.51 }, "framework": "trtllm", "id": "91a43849621ab0af", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 23.14, "tpot_relative": 0.768593, "ttft_error_pct": 60.11, "ttft_relative": 0.398896 }, "aisimulate": { "tpot_error_pct": 18.01, "tpot_relative": 1.180086, "ttft_error_pct": 71.21, "ttft_relative": 0.287931 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.68, "tpot_relative": 1.120067, "ttft_error_pct": 73.38, "ttft_relative": 0.474113 }, "aisimulate": { "tpot_error_pct": 10.5, "tpot_relative": 1.78535, "ttft_error_pct": 73.43, "ttft_relative": 0.473121 }, "concurrency": 256, "measured": { "tpot_relative": 1.61574, "ttft_relative": 1.780867 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.23, "tpot_relative": 1.844787, "ttft_error_pct": 97.52, "ttft_relative": 0.500898 }, "aisimulate": { "tpot_error_pct": 37.01, "tpot_relative": 2.815639, "ttft_error_pct": 95.77, "ttft_relative": 0.855039 }, "concurrency": 512, "measured": { "tpot_relative": 2.055, "ttft_relative": 20.194966 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 35.98, "tpot_shape_error_pct": 2.77, "ttft_mape_pct": 46.77, "ttft_shape_error_pct": 14.5 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 36.95, "tpot_shape_error_pct": 3.83, "ttft_mape_pct": 12.0, "ttft_shape_error_pct": 0.7 }, "framework": "trtllm", "id": "1cf7f1f573972e9e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 38.46, "tpot_relative": 1.384606, "ttft_error_pct": 62.48, "ttft_relative": 1.624752 }, "aisimulate": { "tpot_error_pct": 40.54, "tpot_relative": 1.405367, "ttft_error_pct": 11.89, "ttft_relative": 0.881065 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 38.58, "tpot_relative": 1.83377, "ttft_error_pct": 45.17, "ttft_relative": 1.793284 }, "aisimulate": { "tpot_error_pct": 39.19, "tpot_relative": 1.841827, "ttft_error_pct": 11.44, "ttft_relative": 1.093991 }, "concurrency": 8, "measured": { "tpot_relative": 1.323259, "ttft_relative": 1.235328 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.91, "tpot_relative": 2.420223, "ttft_error_pct": 32.67, "ttft_relative": 2.131456 }, "aisimulate": { "tpot_error_pct": 31.12, "tpot_relative": 2.424216, "ttft_error_pct": 12.68, "ttft_relative": 1.402911 }, "concurrency": 16, "measured": { "tpot_relative": 1.848822, "ttft_relative": 1.606586 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 20.25, "tpot_shape_error_pct": null, "ttft_mape_pct": 23.1, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 22.14, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.56, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "7c41df4ecaab05f9", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 20.25, "tpot_relative": 1.202511, "ttft_error_pct": 23.1, "ttft_relative": 1.230963 }, "aisimulate": { "tpot_error_pct": 22.14, "tpot_relative": 1.221447, "ttft_error_pct": 31.56, "ttft_relative": 0.684441 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 4.5, "tpot_shape_error_pct": 5.27, "ttft_mape_pct": 38.79, "ttft_shape_error_pct": 46.05 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.14, "tpot_shape_error_pct": 5.39, "ttft_mape_pct": 38.55, "ttft_shape_error_pct": 8.65 }, "framework": "vllm", "id": "cc420b7f7f1ce518", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 0.19, "tpot_relative": 1.001859, "ttft_error_pct": 2.12, "ttft_relative": 1.021227 }, "aisimulate": { "tpot_error_pct": 1.36, "tpot_relative": 0.986371, "ttft_error_pct": 41.97, "ttft_relative": 0.580306 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.19, "tpot_relative": 1.172241, "ttft_error_pct": 8.91, "ttft_relative": 1.124462 }, "aisimulate": { "tpot_error_pct": 0.36, "tpot_relative": 1.151271, "ttft_error_pct": 43.7, "ttft_relative": 0.695001 }, "concurrency": 8, "measured": { "tpot_relative": 1.147086, "ttft_relative": 1.234473 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.43, "tpot_relative": 1.522179, "ttft_error_pct": 20.42, "ttft_relative": 1.235182 }, "aisimulate": { "tpot_error_pct": 1.23, "tpot_relative": 1.48985, "ttft_error_pct": 42.08, "ttft_relative": 0.898917 }, "concurrency": 16, "measured": { "tpot_relative": 1.471705, "ttft_relative": 1.552123 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.5, "tpot_relative": 1.972477, "ttft_error_pct": 38.44, "ttft_relative": 1.350996 }, "aisimulate": { "tpot_error_pct": 8.12, "tpot_relative": 1.938234, "ttft_error_pct": 43.19, "ttft_relative": 1.246721 }, "concurrency": 32, "measured": { "tpot_relative": 2.109641, "ttft_relative": 2.194552 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.23, "tpot_relative": 2.938751, "ttft_error_pct": 54.32, "ttft_relative": 1.488853 }, "aisimulate": { "tpot_error_pct": 5.0, "tpot_relative": 2.885009, "ttft_error_pct": 38.82, "ttft_relative": 1.993972 }, "concurrency": 64, "measured": { "tpot_relative": 3.036698, "ttft_relative": 3.259238 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.04, "tpot_relative": 4.299368, "ttft_error_pct": 67.53, "ttft_relative": 1.65288 }, "aisimulate": { "tpot_error_pct": 8.93, "tpot_relative": 4.211676, "ttft_error_pct": 30.67, "ttft_relative": 3.529645 }, "concurrency": 128, "measured": { "tpot_relative": 4.624741, "ttft_relative": 5.091164 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.94, "tpot_relative": 7.029465, "ttft_error_pct": 79.79, "ttft_relative": 1.901043 }, "aisimulate": { "tpot_error_pct": 10.97, "tpot_relative": 6.873084, "ttft_error_pct": 29.38, "ttft_relative": 6.643512 }, "concurrency": 256, "measured": { "tpot_relative": 7.719985, "ttft_relative": 9.40752 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 14.83, "tpot_shape_error_pct": 0.41, "ttft_mape_pct": 14.44, "ttft_shape_error_pct": 10.48 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.37, "tpot_shape_error_pct": 0.62, "ttft_mape_pct": 37.63, "ttft_shape_error_pct": 3.28 }, "framework": "vllm", "id": "5f9e4f7145c8c0e2", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 14.59, "tpot_relative": 1.145945, "ttft_error_pct": 20.77, "ttft_relative": 1.207662 }, "aisimulate": { "tpot_error_pct": 13.02, "tpot_relative": 1.130187, "ttft_error_pct": 36.59, "ttft_relative": 0.634142 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.07, "tpot_relative": 1.326756, "ttft_error_pct": 8.1, "ttft_relative": 1.329878 }, "aisimulate": { "tpot_error_pct": 13.72, "tpot_relative": 1.311257, "ttft_error_pct": 38.67, "ttft_relative": 0.754504 }, "concurrency": 8, "measured": { "tpot_relative": 1.153045, "ttft_relative": 1.230185 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 7.8, "tpot_shape_error_pct": 5.38, "ttft_mape_pct": 37.25, "ttft_shape_error_pct": 47.81 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.17, "tpot_shape_error_pct": 5.57, "ttft_mape_pct": 38.52, "ttft_shape_error_pct": 9.88 }, "framework": "vllm", "id": "b6f6ebcd05fdc668", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 11.46, "tpot_relative": 1.114595, "ttft_error_pct": 9.69, "ttft_relative": 1.096901 }, "aisimulate": { "tpot_error_pct": 8.75, "tpot_relative": 1.087489, "ttft_error_pct": 33.0, "ttft_relative": 0.669998 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.49, "tpot_relative": 1.531722, "ttft_error_pct": 6.04, "ttft_relative": 1.212731 }, "aisimulate": { "tpot_error_pct": 12.43, "tpot_relative": 1.491199, "ttft_error_pct": 37.15, "ttft_relative": 0.811209 }, "concurrency": 8, "measured": { "tpot_relative": 1.326325, "ttft_relative": 1.29072 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.94, "tpot_relative": 2.035232, "ttft_error_pct": 19.46, "ttft_relative": 1.333777 }, "aisimulate": { "tpot_error_pct": 6.24, "tpot_relative": 1.984807, "ttft_error_pct": 36.17, "ttft_relative": 1.05707 }, "concurrency": 16, "measured": { "tpot_relative": 1.868198, "ttft_relative": 1.656003 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.75, "tpot_relative": 2.896757, "ttft_error_pct": 39.52, "ttft_relative": 1.472332 }, "aisimulate": { "tpot_error_pct": 0.06, "tpot_relative": 2.817477, "ttft_error_pct": 39.24, "ttft_relative": 1.479305 }, "concurrency": 32, "measured": { "tpot_relative": 2.819229, "ttft_relative": 2.43461 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.37, "tpot_relative": 4.545296, "ttft_error_pct": 55.91, "ttft_relative": 1.631549 }, "aisimulate": { "tpot_error_pct": 2.87, "tpot_relative": 4.437545, "ttft_error_pct": 35.51, "ttft_relative": 2.386135 }, "concurrency": 64, "measured": { "tpot_relative": 4.313807, "ttft_relative": 3.70023 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.8, "tpot_relative": 6.919122, "ttft_error_pct": 92.86, "ttft_relative": 1.859534 }, "aisimulate": { "tpot_error_pct": 0.65, "tpot_relative": 6.68668, "ttft_error_pct": 50.04, "ttft_relative": 13.005701 }, "concurrency": 128, "measured": { "tpot_relative": 6.730383, "ttft_relative": 26.033989 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 19.2, "tpot_shape_error_pct": null, "ttft_mape_pct": 26.53, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.78, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.58, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "c9255738bc74d162", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 19.2, "tpot_relative": 1.192035, "ttft_error_pct": 26.53, "ttft_relative": 1.26532 }, "aisimulate": { "tpot_error_pct": 17.78, "tpot_relative": 1.177823, "ttft_error_pct": 36.58, "ttft_relative": 0.634151 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 15, "tpot_mape_pct": 7.37, "tpot_shape_error_pct": 5.18, "ttft_mape_pct": 37.48, "ttft_shape_error_pct": 36.03 }, "aisimulate": { "coverage_pct": 100.0, "points": 15, "status_counts": { "failed": 0, "success": 15, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.73, "tpot_shape_error_pct": 4.34, "ttft_mape_pct": 33.98, "ttft_shape_error_pct": 14.83 }, "gpu": "h200", "precisions": [ "fp8" ], "rows": 15, "topologies": [ { "aic": { "points": 9, "tpot_mape_pct": 7.94, "tpot_shape_error_pct": 6.61, "ttft_mape_pct": 40.38, "ttft_shape_error_pct": 35.34 }, "aisimulate": { "points": 9, "status_counts": { "failed": 0, "success": 9, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.37, "tpot_shape_error_pct": 5.12, "ttft_mape_pct": 35.21, "ttft_shape_error_pct": 12.92 }, "framework": "vllm", "id": "f8506771f8420c23", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 13.05, "tpot_relative": 0.869515, "ttft_error_pct": 14.67, "ttft_relative": 0.85326 }, "aisimulate": { "tpot_error_pct": 13.32, "tpot_relative": 0.866775, "ttft_error_pct": 38.12, "ttft_relative": 0.618812 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.36, "tpot_relative": 1.005806, "ttft_error_pct": 9.77, "ttft_relative": 0.973392 }, "aisimulate": { "tpot_error_pct": 11.52, "tpot_relative": 0.981826, "ttft_error_pct": 40.1, "ttft_relative": 0.646151 }, "concurrency": 2, "measured": { "tpot_relative": 1.109617, "ttft_relative": 1.078747 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.62, "tpot_relative": 1.168834, "ttft_error_pct": 16.79, "ttft_relative": 1.082003 }, "aisimulate": { "tpot_error_pct": 7.84, "tpot_relative": 1.14135, "ttft_error_pct": 44.82, "ttft_relative": 0.71754 }, "concurrency": 4, "measured": { "tpot_relative": 1.238464, "ttft_relative": 1.300338 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.72, "tpot_relative": 1.485594, "ttft_error_pct": 19.32, "ttft_relative": 1.192501 }, "aisimulate": { "tpot_error_pct": 5.03, "tpot_relative": 1.450274, "ttft_error_pct": 40.06, "ttft_relative": 0.885959 }, "concurrency": 8, "measured": { "tpot_relative": 1.527145, "ttft_relative": 1.478069 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.8, "tpot_relative": 2.038217, "ttft_error_pct": 37.81, "ttft_relative": 1.307303 }, "aisimulate": { "tpot_error_pct": 3.88, "tpot_relative": 1.995099, "ttft_error_pct": 46.17, "ttft_relative": 1.13159 }, "concurrency": 16, "measured": { "tpot_relative": 2.075616, "ttft_relative": 2.102274 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.28, "tpot_relative": 2.779975, "ttft_error_pct": 46.96, "ttft_relative": 1.427851 }, "aisimulate": { "tpot_error_pct": 11.85, "tpot_relative": 2.731448, "ttft_error_pct": 38.32, "ttft_relative": 1.660297 }, "concurrency": 32, "measured": { "tpot_relative": 3.098489, "ttft_relative": 2.691904 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.25, "tpot_relative": 4.319028, "ttft_error_pct": 61.16, "ttft_relative": 1.558991 }, "aisimulate": { "tpot_error_pct": 8.08, "tpot_relative": 4.280308, "ttft_error_pct": 31.53, "ttft_relative": 2.74798 }, "concurrency": 64, "measured": { "tpot_relative": 4.656488, "ttft_relative": 4.013436 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.11, "tpot_relative": 6.442851, "ttft_error_pct": 73.72, "ttft_relative": 1.727079 }, "aisimulate": { "tpot_error_pct": 11.02, "tpot_relative": 6.377414, "ttft_error_pct": 23.2, "ttft_relative": 5.047027 }, "concurrency": 128, "measured": { "tpot_relative": 7.167284, "ttft_relative": 6.57148 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.27, "tpot_relative": 10.561181, "ttft_error_pct": 83.26, "ttft_relative": 1.956088 }, "aisimulate": { "tpot_error_pct": 11.83, "tpot_relative": 10.495257, "ttft_error_pct": 14.53, "ttft_relative": 9.989092 }, "concurrency": 256, "measured": { "tpot_relative": 11.903195, "ttft_relative": 11.68659 }, "status": "success" } ], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 6.51, "tpot_shape_error_pct": 2.89, "ttft_mape_pct": 33.13, "ttft_shape_error_pct": 37.13 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.77, "tpot_shape_error_pct": 3.08, "ttft_mape_pct": 32.15, "ttft_shape_error_pct": 17.87 }, "framework": "vllm", "id": "f3a9f36025d31f67", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 6.06, "tpot_relative": 0.9394, "ttft_error_pct": 6.13, "ttft_relative": 0.938708 }, "aisimulate": { "tpot_error_pct": 7.93, "tpot_relative": 0.920713, "ttft_error_pct": 40.95, "ttft_relative": 0.590521 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.4, "tpot_relative": 1.120696, "ttft_error_pct": 2.14, "ttft_relative": 1.033388 }, "aisimulate": { "tpot_error_pct": 7.24, "tpot_relative": 1.098879, "ttft_error_pct": 28.03, "ttft_relative": 0.728143 }, "concurrency": 8, "measured": { "tpot_relative": 1.18468, "ttft_relative": 1.011753 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.14, "tpot_relative": 1.435308, "ttft_error_pct": 15.36, "ttft_relative": 1.130109 }, "aisimulate": { "tpot_error_pct": 6.8, "tpot_relative": 1.410208, "ttft_error_pct": 30.81, "ttft_relative": 0.92382 }, "concurrency": 16, "measured": { "tpot_relative": 1.513055, "ttft_relative": 1.335215 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.46, "tpot_relative": 1.968314, "ttft_error_pct": 43.46, "ttft_relative": 1.23204 }, "aisimulate": { "tpot_error_pct": 5.84, "tpot_relative": 1.939884, "ttft_error_pct": 37.82, "ttft_relative": 1.355069 }, "concurrency": 32, "measured": { "tpot_relative": 2.060149, "ttft_relative": 2.17911 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.81, "tpot_relative": 2.928685, "ttft_error_pct": 59.29, "ttft_relative": 1.339911 }, "aisimulate": { "tpot_error_pct": 4.26, "tpot_relative": 2.914925, "ttft_error_pct": 32.16, "ttft_relative": 2.232816 }, "concurrency": 64, "measured": { "tpot_relative": 3.044726, "ttft_relative": 3.291409 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.19, "tpot_relative": 4.136248, "ttft_error_pct": 72.4, "ttft_relative": 1.462988 }, "aisimulate": { "tpot_error_pct": 14.53, "tpot_relative": 4.119717, "ttft_error_pct": 23.16, "ttft_relative": 4.07279 }, "concurrency": 128, "measured": { "tpot_relative": 4.820115, "ttft_relative": 5.300125 }, "status": "success" } ], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4", "fp8" ], "rows": 88 } ] }, { "aic": { "points": 115, "tpot_mape_pct": 44.78, "tpot_shape_error_pct": 31.94, "ttft_mape_pct": 42.44, "ttft_shape_error_pct": 46.26 }, "aisimulate": { "coverage_pct": 100.0, "points": 115, "status_counts": { "failed": 0, "success": 115, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 41.72, "tpot_shape_error_pct": 30.05, "ttft_mape_pct": 31.49, "ttft_shape_error_pct": 39.16 }, "frameworks": [ "dynamo-vllm", "vllm" ], "gpu_skus": [ "b200", "b300" ], "hf_model_paths": [ "MiniMaxAI/MiniMax-M3" ], "model": "MiniMax-M3", "precisions": [ "fp4" ], "rows": 115, "workloads": [ { "aic": { "points": 45, "tpot_mape_pct": 38.01, "tpot_shape_error_pct": 33.96, "ttft_mape_pct": 48.24, "ttft_shape_error_pct": 50.25 }, "aisimulate": { "coverage_pct": 100.0, "points": 45, "status_counts": { "failed": 0, "success": 45, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 33.2, "tpot_shape_error_pct": 32.19, "ttft_mape_pct": 39.9, "ttft_shape_error_pct": 28.89 }, "gpu_skus": [ "b200", "b300" ], "gpus": [ { "aic": { "points": 17, "tpot_mape_pct": 26.68, "tpot_shape_error_pct": 23.42, "ttft_mape_pct": 45.56, "ttft_shape_error_pct": 56.88 }, "aisimulate": { "coverage_pct": 100.0, "points": 17, "status_counts": { "failed": 0, "success": 17, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 23.81, "tpot_shape_error_pct": 21.4, "ttft_mape_pct": 42.93, "ttft_shape_error_pct": 35.75 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 17, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 20.36, "tpot_shape_error_pct": 31.38, "ttft_mape_pct": 45.47, "ttft_shape_error_pct": 60.88 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 28.87, "tpot_shape_error_pct": 27.64, "ttft_mape_pct": 35.92, "ttft_shape_error_pct": 21.78 }, "framework": "vllm", "id": "df914459bdf089df", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 57.41, "tpot_relative": 1.574104, "ttft_error_pct": 4.47, "ttft_relative": 1.044748 }, "aisimulate": { "tpot_error_pct": 62.57, "tpot_relative": 1.625685, "ttft_error_pct": 23.41, "ttft_relative": 0.765914 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.57, "tpot_relative": 1.865649, "ttft_error_pct": 37.24, "ttft_relative": 1.138153 }, "aisimulate": { "tpot_error_pct": 22.21, "tpot_relative": 1.989909, "ttft_error_pct": 36.48, "ttft_relative": 1.152003 }, "concurrency": 128, "measured": { "tpot_relative": 1.628336, "ttft_relative": 1.813589 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.14, "tpot_relative": 2.197148, "ttft_error_pct": 62.07, "ttft_relative": 1.245569 }, "aisimulate": { "tpot_error_pct": 18.49, "tpot_relative": 2.407457, "ttft_error_pct": 41.63, "ttft_relative": 1.916873 }, "concurrency": 256, "measured": { "tpot_relative": 2.031781, "ttft_relative": 3.284031 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.31, "tpot_relative": 2.875565, "ttft_error_pct": 78.08, "ttft_relative": 1.317527 }, "aisimulate": { "tpot_error_pct": 12.23, "tpot_relative": 3.185377, "ttft_error_pct": 42.16, "ttft_relative": 3.477038 }, "concurrency": 512, "measured": { "tpot_relative": 2.83835, "ttft_relative": 6.011604 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 39.08, "tpot_shape_error_pct": 31.08, "ttft_mape_pct": 47.91, "ttft_shape_error_pct": 54.96 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.84, "tpot_shape_error_pct": 27.61, "ttft_mape_pct": 37.18, "ttft_shape_error_pct": 28.08 }, "framework": "vllm", "id": "b67bdc595ea0cfc4", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 23.16, "tpot_relative": 0.768407, "ttft_error_pct": 17.78, "ttft_relative": 0.822185 }, "aisimulate": { "tpot_error_pct": 17.18, "tpot_relative": 1.171826, "ttft_error_pct": 22.72, "ttft_relative": 0.772801 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 51.71, "tpot_relative": 0.932123, "ttft_error_pct": 58.1, "ttft_relative": 0.90318 }, "aisimulate": { "tpot_error_pct": 19.11, "tpot_relative": 1.561363, "ttft_error_pct": 47.58, "ttft_relative": 1.130033 }, "concurrency": 512, "measured": { "tpot_relative": 1.93019, "ttft_relative": 2.15575 }, "status": "success" }, { "aic": { "tpot_error_pct": 42.37, "tpot_relative": 1.41749, "ttft_error_pct": 67.84, "ttft_relative": 1.020281 }, "aisimulate": { "tpot_error_pct": 11.22, "tpot_relative": 2.183669, "ttft_error_pct": 41.25, "ttft_relative": 1.863828 }, "concurrency": 1024, "measured": { "tpot_relative": 2.459758, "ttft_relative": 3.172653 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 10, "tpot_mape_pct": 25.49, "tpot_shape_error_pct": 19.07, "ttft_mape_pct": 44.9, "ttft_shape_error_pct": 55.97 }, "aisimulate": { "points": 10, "status_counts": { "failed": 0, "success": 10, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 24.17, "tpot_shape_error_pct": 17.94, "ttft_mape_pct": 47.45, "ttft_shape_error_pct": 42.11 }, "framework": "vllm", "id": "6ab9340fa177e031", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 16.76, "tpot_relative": 1.167577, "ttft_error_pct": 88.32, "ttft_relative": 1.88316 }, "aisimulate": { "tpot_error_pct": 16.62, "tpot_relative": 1.166217, "ttft_error_pct": 15.38, "ttft_relative": 0.846154 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.48, "tpot_relative": 1.184587, "ttft_error_pct": 50.27, "ttft_relative": 2.170509 }, "aisimulate": { "tpot_error_pct": 10.1, "tpot_relative": 1.180439, "ttft_error_pct": 39.93, "ttft_relative": 0.8677 }, "concurrency": 2, "measured": { "tpot_relative": 1.07218, "ttft_relative": 1.444447 }, "status": "success" }, { "aic": { "tpot_error_pct": 34.66, "tpot_relative": 1.600065, "ttft_error_pct": 11.68, "ttft_relative": 2.411881 }, "aisimulate": { "tpot_error_pct": 33.82, "tpot_relative": 1.590073, "ttft_error_pct": 64.28, "ttft_relative": 0.975367 }, "concurrency": 4, "measured": { "tpot_relative": 1.188231, "ttft_relative": 2.730872 }, "status": "success" }, { "aic": { "tpot_error_pct": 34.86, "tpot_relative": 1.836677, "ttft_error_pct": 15.33, "ttft_relative": 2.653895 }, "aisimulate": { "tpot_error_pct": 34.79, "tpot_relative": 1.835819, "ttft_error_pct": 51.71, "ttft_relative": 1.111321 }, "concurrency": 8, "measured": { "tpot_relative": 1.361951, "ttft_relative": 2.301121 }, "status": "success" }, { "aic": { "tpot_error_pct": 50.21, "tpot_relative": 2.406111, "ttft_error_pct": 37.85, "ttft_relative": 2.899023 }, "aisimulate": { "tpot_error_pct": 50.67, "tpot_relative": 2.41352, "ttft_error_pct": 37.22, "ttft_relative": 1.320379 }, "concurrency": 16, "measured": { "tpot_relative": 1.601851, "ttft_relative": 2.103019 }, "status": "success" }, { "aic": { "tpot_error_pct": 49.29, "tpot_relative": 2.907105, "ttft_error_pct": 20.52, "ttft_relative": 3.15413 }, "aisimulate": { "tpot_error_pct": 51.62, "tpot_relative": 2.952515, "ttft_error_pct": 57.91, "ttft_relative": 1.670286 }, "concurrency": 32, "measured": { "tpot_relative": 1.947305, "ttft_relative": 3.968607 }, "status": "success" }, { "aic": { "tpot_error_pct": 29.45, "tpot_relative": 3.166362, "ttft_error_pct": 32.26, "ttft_relative": 3.429301 }, "aisimulate": { "tpot_error_pct": 33.5, "tpot_relative": 3.265467, "ttft_error_pct": 55.5, "ttft_relative": 2.252876 }, "concurrency": 64, "measured": { "tpot_relative": 2.446102, "ttft_relative": 5.062661 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.78, "tpot_relative": 3.876725, "ttft_error_pct": 48.52, "ttft_relative": 3.743564 }, "aisimulate": { "tpot_error_pct": 1.29, "tpot_relative": 4.105245, "ttft_error_pct": 52.49, "ttft_relative": 3.454916 }, "concurrency": 128, "measured": { "tpot_relative": 4.158707, "ttft_relative": 7.271284 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.35, "tpot_relative": 4.47948, "ttft_error_pct": 64.97, "ttft_relative": 4.097277 }, "aisimulate": { "tpot_error_pct": 5.72, "tpot_relative": 4.818564, "ttft_error_pct": 50.55, "ttft_relative": 5.78415 }, "concurrency": 256, "measured": { "tpot_relative": 5.110678, "ttft_relative": 11.697237 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.05, "tpot_relative": 6.072325, "ttft_error_pct": 79.23, "ttft_relative": 4.348608 }, "aisimulate": { "tpot_error_pct": 3.57, "tpot_relative": 6.509558, "ttft_error_pct": 49.54, "ttft_relative": 10.567326 }, "concurrency": 512, "measured": { "tpot_relative": 6.750628, "ttft_relative": 20.941967 }, "status": "success" } ], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 28, "tpot_mape_pct": 44.89, "tpot_shape_error_pct": 40.66, "ttft_mape_pct": 49.86, "ttft_shape_error_pct": 46.02 }, "aisimulate": { "coverage_pct": 100.0, "points": 28, "status_counts": { "failed": 0, "success": 28, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 38.9, "tpot_shape_error_pct": 39.06, "ttft_mape_pct": 38.06, "ttft_shape_error_pct": 24.52 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 28, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 24.62, "tpot_shape_error_pct": 29.24, "ttft_mape_pct": 27.37, "ttft_shape_error_pct": 38.98 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 17.51, "tpot_shape_error_pct": 28.43, "ttft_mape_pct": 46.8, "ttft_shape_error_pct": 16.03 }, "framework": "dynamo-vllm", "id": "eab724a790cc85f8", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 2.21, "tpot_relative": 1.022076, "ttft_error_pct": 9.0, "ttft_relative": 1.090046 }, "aisimulate": { "tpot_error_pct": 5.34, "tpot_relative": 0.946591, "ttft_error_pct": 39.53, "ttft_relative": 0.60471 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.24, "tpot_relative": 1.380997, "ttft_error_pct": 31.18, "ttft_relative": 1.090046 }, "aisimulate": { "tpot_error_pct": 15.2, "tpot_relative": 1.270237, "ttft_error_pct": 53.4, "ttft_relative": 0.738125 }, "concurrency": 4, "measured": { "tpot_relative": 1.102665, "ttft_relative": 1.583794 }, "status": "success" }, { "aic": { "tpot_error_pct": 36.28, "tpot_relative": 1.681535, "ttft_error_pct": 28.98, "ttft_relative": 1.090046 }, "aisimulate": { "tpot_error_pct": 25.54, "tpot_relative": 1.549006, "ttft_error_pct": 47.37, "ttft_relative": 0.807721 }, "concurrency": 8, "measured": { "tpot_relative": 1.23385, "ttft_relative": 1.534821 }, "status": "success" }, { "aic": { "tpot_error_pct": 34.74, "tpot_relative": 1.981303, "ttft_error_pct": 40.3, "ttft_relative": 1.090046 }, "aisimulate": { "tpot_error_pct": 23.98, "tpot_relative": 1.823006, "ttft_error_pct": 46.9, "ttft_relative": 0.969562 }, "concurrency": 16, "measured": { "tpot_relative": 1.470452, "ttft_relative": 1.826011 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 10, "tpot_mape_pct": 79.65, "tpot_shape_error_pct": 67.24, "ttft_mape_pct": 68.7, "ttft_shape_error_pct": 47.35 }, "aisimulate": { "points": 10, "status_counts": { "failed": 0, "success": 10, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 79.42, "tpot_shape_error_pct": 67.11, "ttft_mape_pct": 36.56, "ttft_shape_error_pct": 36.94 }, "framework": "vllm", "id": "0b4dc7a10323e0a5", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 21.18, "tpot_relative": 1.211783, "ttft_error_pct": 123.56, "ttft_relative": 2.235622 }, "aisimulate": { "tpot_error_pct": 21.21, "tpot_relative": 1.212129, "ttft_error_pct": 7.1, "ttft_relative": 0.92903 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.62, "tpot_relative": 1.213178, "ttft_error_pct": 107.91, "ttft_relative": 2.575305 }, "aisimulate": { "tpot_error_pct": 11.38, "tpot_relative": 1.210569, "ttft_error_pct": 22.87, "ttft_relative": 0.955393 }, "concurrency": 2, "measured": { "tpot_relative": 1.086851, "ttft_relative": 1.238685 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.26, "tpot_relative": 1.452865, "ttft_error_pct": 47.08, "ttft_relative": 2.863227 }, "aisimulate": { "tpot_error_pct": 19.63, "tpot_relative": 1.445281, "ttft_error_pct": 46.08, "ttft_relative": 1.049693 }, "concurrency": 4, "measured": { "tpot_relative": 1.20814, "ttft_relative": 1.946733 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.69, "tpot_relative": 1.730003, "ttft_error_pct": 110.63, "ttft_relative": 3.15049 }, "aisimulate": { "tpot_error_pct": 30.63, "tpot_relative": 1.72917, "ttft_error_pct": 18.81, "ttft_relative": 1.214389 }, "concurrency": 8, "measured": { "tpot_relative": 1.323703, "ttft_relative": 1.495771 }, "status": "success" }, { "aic": { "tpot_error_pct": 614.67, "tpot_relative": 11.435185, "ttft_error_pct": 75.16, "ttft_relative": 3.439864 }, "aisimulate": { "tpot_error_pct": 613.55, "tpot_relative": 11.41724, "ttft_error_pct": 44.5, "ttft_relative": 2.83779 }, "concurrency": 16, "measured": { "tpot_relative": 1.60007, "ttft_relative": 1.963811 }, "status": "success" }, { "aic": { "tpot_error_pct": 34.06, "tpot_relative": 2.574534, "ttft_error_pct": 20.63, "ttft_relative": 3.744901 }, "aisimulate": { "tpot_error_pct": 42.94, "tpot_relative": 2.745029, "ttft_error_pct": 37.24, "ttft_relative": 1.948189 }, "concurrency": 32, "measured": { "tpot_relative": 1.920365, "ttft_relative": 3.104358 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.88, "tpot_relative": 2.997671, "ttft_error_pct": 17.51, "ttft_relative": 4.077489 }, "aisimulate": { "tpot_error_pct": 26.09, "tpot_relative": 3.12666, "ttft_error_pct": 46.09, "ttft_relative": 2.665038 }, "concurrency": 64, "measured": { "tpot_relative": 2.479791, "ttft_relative": 4.943083 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.51, "tpot_relative": 3.490558, "ttft_error_pct": 50.5, "ttft_relative": 4.463955 }, "aisimulate": { "tpot_error_pct": 6.38, "tpot_relative": 3.692622, "ttft_error_pct": 55.51, "ttft_relative": 4.012442 }, "concurrency": 128, "measured": { "tpot_relative": 3.944455, "ttft_relative": 9.018598 }, "status": "success" }, { "aic": { "tpot_error_pct": 19.01, "tpot_relative": 4.216163, "ttft_error_pct": 58.05, "ttft_relative": 4.915673 }, "aisimulate": { "tpot_error_pct": 13.88, "tpot_relative": 4.483055, "ttft_error_pct": 43.02, "ttft_relative": 6.677986 }, "concurrency": 256, "measured": { "tpot_relative": 5.205553, "ttft_relative": 11.718956 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.61, "tpot_relative": 5.91211, "ttft_error_pct": 75.92, "ttft_relative": 5.243745 }, "aisimulate": { "tpot_error_pct": 8.46, "tpot_relative": 6.192697, "ttft_error_pct": 44.41, "ttft_relative": 12.103955 }, "concurrency": 512, "measured": { "tpot_relative": 6.764992, "ttft_relative": 21.773883 }, "status": "success" } ], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 13.63, "tpot_shape_error_pct": 28.38, "ttft_mape_pct": 52.93, "ttft_shape_error_pct": 49.26 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 10.32, "tpot_shape_error_pct": 25.6, "ttft_mape_pct": 47.29, "ttft_shape_error_pct": 6.78 }, "framework": "vllm", "id": "538ada49e38a545e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 26.68, "tpot_relative": 1.266775, "ttft_error_pct": 25.35, "ttft_relative": 0.746497 }, "aisimulate": { "tpot_error_pct": 30.55, "tpot_relative": 1.305467, "ttft_error_pct": 48.71, "ttft_relative": 0.512923 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.84, "tpot_relative": 1.482559, "ttft_error_pct": 47.3, "ttft_relative": 0.815156 }, "aisimulate": { "tpot_error_pct": 1.05, "tpot_relative": 1.574308, "ttft_error_pct": 49.72, "ttft_relative": 0.777701 }, "concurrency": 128, "measured": { "tpot_relative": 1.557988, "ttft_relative": 1.546732 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.0, "tpot_relative": 1.778219, "ttft_error_pct": 59.18, "ttft_relative": 0.897618 }, "aisimulate": { "tpot_error_pct": 1.78, "tpot_relative": 1.919206, "ttft_error_pct": 40.65, "ttft_relative": 1.305269 }, "concurrency": 256, "measured": { "tpot_relative": 1.954073, "ttft_relative": 2.19917 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.99, "tpot_relative": 2.430781, "ttft_error_pct": 79.88, "ttft_relative": 0.959083 }, "aisimulate": { "tpot_error_pct": 7.89, "tpot_relative": 2.603229, "ttft_error_pct": 50.07, "ttft_relative": 2.380546 }, "concurrency": 512, "measured": { "tpot_relative": 2.826219, "ttft_relative": 4.767868 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 21.52, "tpot_shape_error_pct": 24.35, "ttft_mape_pct": 36.71, "ttft_shape_error_pct": 51.58 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.8, "tpot_shape_error_pct": 16.72, "ttft_mape_pct": 38.61, "ttft_shape_error_pct": 26.87 }, "framework": "vllm", "id": "d645fe3d6d2db0bc", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 6.32, "tpot_relative": 0.936829, "ttft_error_pct": 3.54, "ttft_relative": 0.964614 }, "aisimulate": { "tpot_error_pct": 12.76, "tpot_relative": 1.12765, "ttft_error_pct": 25.22, "ttft_relative": 0.747838 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.75, "tpot_relative": 1.070588, "ttft_error_pct": 42.9, "ttft_relative": 1.056964 }, "aisimulate": { "tpot_error_pct": 0.73, "tpot_relative": 1.395848, "ttft_error_pct": 42.85, "ttft_relative": 1.057863 }, "concurrency": 256, "measured": { "tpot_relative": 1.385789, "ttft_relative": 1.851092 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.51, "tpot_relative": 1.410118, "ttft_error_pct": 63.68, "ttft_relative": 1.175462 }, "aisimulate": { "tpot_error_pct": 12.91, "tpot_relative": 1.904139, "ttft_error_pct": 47.77, "ttft_relative": 1.690674 }, "concurrency": 512, "measured": { "tpot_relative": 2.186448, "ttft_relative": 3.236781 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 41.22, "tpot_shape_error_pct": 17.65, "ttft_mape_pct": 44.02, "ttft_shape_error_pct": 47.94 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 8.24, "tpot_shape_error_pct": 13.94, "ttft_mape_pct": 37.15, "ttft_shape_error_pct": 17.51 }, "framework": "vllm", "id": "e90960e545221bf8", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 33.38, "tpot_relative": 0.666194, "ttft_error_pct": 17.73, "ttft_relative": 0.822738 }, "aisimulate": { "tpot_error_pct": 4.41, "tpot_relative": 1.044074, "ttft_error_pct": 28.84, "ttft_relative": 0.711555 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 47.81, "tpot_relative": 0.84667, "ttft_error_pct": 48.94, "ttft_relative": 0.90233 }, "aisimulate": { "tpot_error_pct": 11.7, "tpot_relative": 1.432424, "ttft_error_pct": 41.22, "ttft_relative": 1.038655 }, "concurrency": 512, "measured": { "tpot_relative": 1.622276, "ttft_relative": 1.767122 }, "status": "success" }, { "aic": { "tpot_error_pct": 42.47, "tpot_relative": 1.239382, "ttft_error_pct": 65.4, "ttft_relative": 1.004828 }, "aisimulate": { "tpot_error_pct": 8.6, "tpot_relative": 1.969077, "ttft_error_pct": 41.38, "ttft_relative": 1.702396 }, "concurrency": 1024, "measured": { "tpot_relative": 2.154379, "ttft_relative": 2.904021 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 29.81, "tpot_shape_error_pct": 10.87, "ttft_mape_pct": 36.47, "ttft_shape_error_pct": 40.89 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 33.14, "tpot_shape_error_pct": 10.66, "ttft_mape_pct": 24.13, "ttft_shape_error_pct": 16.6 }, "framework": "vllm", "id": "31e27cf33a9e4600", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 36.39, "tpot_relative": 1.363939, "ttft_error_pct": 58.17, "ttft_relative": 1.581668 }, "aisimulate": { "tpot_error_pct": 37.09, "tpot_relative": 1.370894, "ttft_error_pct": 13.34, "ttft_relative": 0.866637 }, "concurrency": 16, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 40.91, "tpot_relative": 1.809222, "ttft_error_pct": 34.1, "ttft_relative": 1.726449 }, "aisimulate": { "tpot_error_pct": 42.35, "tpot_relative": 1.827609, "ttft_error_pct": 14.83, "ttft_relative": 1.096511 }, "concurrency": 32, "measured": { "tpot_relative": 1.283921, "ttft_relative": 1.287446 }, "status": "success" }, { "aic": { "tpot_error_pct": 40.94, "tpot_relative": 2.344525, "ttft_error_pct": 1.39, "ttft_relative": 1.883664 }, "aisimulate": { "tpot_error_pct": 45.85, "tpot_relative": 2.426123, "ttft_error_pct": 21.01, "ttft_relative": 1.508894 }, "concurrency": 64, "measured": { "tpot_relative": 1.663485, "ttft_relative": 1.910271 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.99, "tpot_relative": 2.753507, "ttft_error_pct": 52.23, "ttft_relative": 2.054276 }, "aisimulate": { "tpot_error_pct": 7.26, "tpot_relative": 2.9244, "ttft_error_pct": 47.34, "ttft_relative": 2.264691 }, "concurrency": 128, "measured": { "tpot_relative": 2.726407, "ttft_relative": 4.300303 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "fp4" ], "rows": 45 }, { "aic": { "points": 70, "tpot_mape_pct": 49.14, "tpot_shape_error_pct": 30.54, "ttft_mape_pct": 38.72, "ttft_shape_error_pct": 43.5 }, "aisimulate": { "coverage_pct": 100.0, "points": 70, "status_counts": { "failed": 0, "success": 70, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 47.19, "tpot_shape_error_pct": 28.56, "ttft_mape_pct": 26.09, "ttft_shape_error_pct": 46.27 }, "gpu_skus": [ "b200", "b300" ], "gpus": [ { "aic": { "points": 33, "tpot_mape_pct": 46.59, "tpot_shape_error_pct": 33.42, "ttft_mape_pct": 36.84, "ttft_shape_error_pct": 42.93 }, "aisimulate": { "coverage_pct": 100.0, "points": 33, "status_counts": { "failed": 0, "success": 33, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 47.43, "tpot_shape_error_pct": 31.2, "ttft_mape_pct": 26.94, "ttft_shape_error_pct": 51.03 }, "gpu": "b200", "precisions": [ "fp4" ], "rows": 33, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 82.65, "tpot_shape_error_pct": 20.1, "ttft_mape_pct": 26.32, "ttft_shape_error_pct": 38.39 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 67.39, "tpot_shape_error_pct": 18.5, "ttft_mape_pct": 44.51, "ttft_shape_error_pct": 11.8 }, "framework": "dynamo-vllm", "id": "26d7881d9f19e6fb", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 58.72, "tpot_relative": 1.587203, "ttft_error_pct": 11.64, "ttft_relative": 1.116404 }, "aisimulate": { "tpot_error_pct": 46.99, "tpot_relative": 1.46993, "ttft_error_pct": 45.67, "ttft_relative": 0.543311 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 82.85, "tpot_relative": 2.221225, "ttft_error_pct": 20.05, "ttft_relative": 1.116404 }, "aisimulate": { "tpot_error_pct": 67.22, "tpot_relative": 2.031281, "ttft_error_pct": 52.64, "ttft_relative": 0.661234 }, "concurrency": 4, "measured": { "tpot_relative": 1.214759, "ttft_relative": 1.396309 }, "status": "success" }, { "aic": { "tpot_error_pct": 96.45, "tpot_relative": 2.770254, "ttft_error_pct": 26.34, "ttft_relative": 1.116404 }, "aisimulate": { "tpot_error_pct": 79.8, "tpot_relative": 2.535412, "ttft_error_pct": 33.72, "ttft_relative": 1.004451 }, "concurrency": 8, "measured": { "tpot_relative": 1.410155, "ttft_relative": 1.515565 }, "status": "success" }, { "aic": { "tpot_error_pct": 92.58, "tpot_relative": 3.531768, "ttft_error_pct": 47.27, "ttft_relative": 1.116404 }, "aisimulate": { "tpot_error_pct": 75.54, "tpot_relative": 3.219235, "ttft_error_pct": 45.98, "ttft_relative": 1.143702 }, "concurrency": 16, "measured": { "tpot_relative": 1.833911, "ttft_relative": 2.117314 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 18.23, "tpot_shape_error_pct": 11.42, "ttft_mape_pct": 65.41, "ttft_shape_error_pct": 45.73 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 22.91, "tpot_shape_error_pct": 8.89, "ttft_mape_pct": 8.4, "ttft_shape_error_pct": 18.46 }, "framework": "vllm", "id": "3b5c3f55eb0332f9", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 27.97, "tpot_relative": 1.279666, "ttft_error_pct": 50.23, "ttft_relative": 0.497687 }, "aisimulate": { "tpot_error_pct": 30.65, "tpot_relative": 1.306522, "ttft_error_pct": 8.56, "ttft_relative": 0.914421 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.78, "tpot_relative": 1.737749, "ttft_error_pct": 66.4, "ttft_relative": 0.540269 }, "aisimulate": { "tpot_error_pct": 23.53, "tpot_relative": 1.807192, "ttft_error_pct": 4.04, "ttft_relative": 1.672771 }, "concurrency": 128, "measured": { "tpot_relative": 1.462942, "ttft_relative": 1.60786 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.93, "tpot_relative": 2.532772, "ttft_error_pct": 79.59, "ttft_relative": 0.591705 }, "aisimulate": { "tpot_error_pct": 14.54, "tpot_relative": 2.687928, "ttft_error_pct": 12.61, "ttft_relative": 3.264375 }, "concurrency": 256, "measured": { "tpot_relative": 2.34667, "ttft_relative": 2.898775 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 4, "tpot_mape_pct": 48.87, "tpot_shape_error_pct": 11.47, "ttft_mape_pct": 25.41, "ttft_shape_error_pct": 28.02 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 48.83, "tpot_shape_error_pct": 8.71, "ttft_mape_pct": 23.79, "ttft_shape_error_pct": 22.06 }, "framework": "vllm", "id": "00391c607783c0da", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 51.3, "tpot_relative": 1.512951, "ttft_error_pct": 6.18, "ttft_relative": 1.061807 }, "aisimulate": { "tpot_error_pct": 51.33, "tpot_relative": 1.513269, "ttft_error_pct": 21.66, "ttft_relative": 0.783393 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 56.14, "tpot_relative": 1.750833, "ttft_error_pct": 9.79, "ttft_relative": 1.206635 }, "aisimulate": { "tpot_error_pct": 52.95, "tpot_relative": 1.715067, "ttft_error_pct": 25.18, "ttft_relative": 0.822345 }, "concurrency": 2, "measured": { "tpot_relative": 1.121293, "ttft_relative": 1.099072 }, "status": "success" }, { "aic": { "tpot_error_pct": 67.63, "tpot_relative": 2.241136, "ttft_error_pct": 7.63, "ttft_relative": 1.341221 }, "aisimulate": { "tpot_error_pct": 64.5, "tpot_relative": 2.199257, "ttft_error_pct": 26.0, "ttft_relative": 0.922099 }, "concurrency": 4, "measured": { "tpot_relative": 1.336965, "ttft_relative": 1.246162 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.42, "tpot_relative": 14.616027, "ttft_error_pct": 78.03, "ttft_relative": 2.261217 }, "aisimulate": { "tpot_error_pct": 26.56, "tpot_relative": 15.360913, "ttft_error_pct": 22.33, "ttft_relative": 12.59075 }, "concurrency": 256, "measured": { "tpot_relative": 12.137083, "ttft_relative": 10.292371 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 35.52, "tpot_shape_error_pct": 17.85, "ttft_mape_pct": 13.86, "ttft_shape_error_pct": 26.06 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 76.28, "tpot_shape_error_pct": 2.28, "ttft_mape_pct": 2.06, "ttft_shape_error_pct": 0.95 }, "framework": "vllm", "id": "e0b20caebaeb0134", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 29.2, "tpot_relative": 0.70801, "ttft_error_pct": 0.95, "ttft_relative": 0.990489 }, "aisimulate": { "tpot_error_pct": 78.32, "tpot_relative": 1.783182, "ttft_error_pct": 2.55, "ttft_relative": 1.025452 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 41.84, "tpot_relative": 0.933644, "ttft_error_pct": 26.77, "ttft_relative": 1.076579 }, "aisimulate": { "tpot_error_pct": 74.24, "tpot_relative": 2.797103, "ttft_error_pct": 1.57, "ttft_relative": 1.493106 }, "concurrency": 256, "measured": { "tpot_relative": 1.605272, "ttft_relative": 1.470071 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 9, "tpot_mape_pct": 17.3, "tpot_shape_error_pct": 13.44, "ttft_mape_pct": 45.82, "ttft_shape_error_pct": 35.44 }, "aisimulate": { "points": 9, "status_counts": { "failed": 0, "success": 9, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 16.18, "tpot_shape_error_pct": 12.41, "ttft_mape_pct": 36.1, "ttft_shape_error_pct": 97.88 }, "framework": "vllm", "id": "ded5303912fe1bb4", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 12.93, "tpot_relative": 1.129323, "ttft_error_pct": 50.56, "ttft_relative": 0.494367 }, "aisimulate": { "tpot_error_pct": 12.9, "tpot_relative": 1.128954, "ttft_error_pct": 65.83, "ttft_relative": 0.341703 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.21, "tpot_relative": 1.177247, "ttft_error_pct": 37.49, "ttft_relative": 0.562674 }, "aisimulate": { "tpot_error_pct": 3.43, "tpot_relative": 1.157307, "ttft_error_pct": 60.34, "ttft_relative": 0.35697 }, "concurrency": 2, "measured": { "tpot_relative": 1.118923, "ttft_relative": 0.900174 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.77, "tpot_relative": 1.627811, "ttft_error_pct": 22.11, "ttft_relative": 0.625525 }, "aisimulate": { "tpot_error_pct": 28.57, "tpot_relative": 1.600414, "ttft_error_pct": 49.78, "ttft_relative": 0.403351 }, "concurrency": 4, "measured": { "tpot_relative": 1.244806, "ttft_relative": 0.803125 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.22, "tpot_relative": 1.915806, "ttft_error_pct": 28.79, "ttft_relative": 0.688508 }, "aisimulate": { "tpot_error_pct": 22.75, "tpot_relative": 1.893208, "ttft_error_pct": 48.33, "ttft_relative": 0.499567 }, "concurrency": 8, "measured": { "tpot_relative": 1.542322, "ttft_relative": 0.966896 }, "status": "success" }, { "aic": { "tpot_error_pct": 34.56, "tpot_relative": 2.595056, "ttft_error_pct": 26.9, "ttft_relative": 0.752026 }, "aisimulate": { "tpot_error_pct": 33.67, "tpot_relative": 2.577947, "ttft_error_pct": 38.1, "ttft_relative": 0.636813 }, "concurrency": 16, "measured": { "tpot_relative": 1.9286, "ttft_relative": 1.02882 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.42, "tpot_relative": 3.403948, "ttft_error_pct": 46.1, "ttft_relative": 0.819227 }, "aisimulate": { "tpot_error_pct": 23.52, "tpot_relative": 3.406797, "ttft_error_pct": 38.25, "ttft_relative": 0.938442 }, "concurrency": 32, "measured": { "tpot_relative": 2.75806, "ttft_relative": 1.519791 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.46, "tpot_relative": 4.23375, "ttft_error_pct": 52.25, "ttft_relative": 0.890593 }, "aisimulate": { "tpot_error_pct": 6.91, "tpot_relative": 4.291917, "ttft_error_pct": 17.47, "ttft_relative": 1.539264 }, "concurrency": 64, "measured": { "tpot_relative": 4.014612, "ttft_relative": 1.86499 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.72, "tpot_relative": 5.965284, "ttft_error_pct": 67.61, "ttft_relative": 0.96811 }, "aisimulate": { "tpot_error_pct": 0.32, "tpot_relative": 6.111945, "ttft_error_pct": 6.16, "ttft_relative": 2.805214 }, "concurrency": 128, "measured": { "tpot_relative": 6.131794, "ttft_relative": 2.989329 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.38, "tpot_relative": 8.843896, "ttft_error_pct": 80.59, "ttft_relative": 1.063479 }, "aisimulate": { "tpot_error_pct": 13.55, "tpot_relative": 9.143466, "ttft_error_pct": 0.63, "ttft_relative": 5.444153 }, "concurrency": 256, "measured": { "tpot_relative": 10.576233, "ttft_relative": 5.47855 }, "status": "success" } ], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 69.68, "tpot_shape_error_pct": 237.73, "ttft_mape_pct": 55.78, "ttft_shape_error_pct": 173.74 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 68.76, "tpot_shape_error_pct": 232.99, "ttft_mape_pct": 70.44, "ttft_shape_error_pct": 159.66 }, "framework": "vllm", "id": "2cbefba2409d0bb0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 70.95, "tpot_relative": 0.290458, "ttft_error_pct": 78.84, "ttft_relative": 0.211615 }, "aisimulate": { "tpot_error_pct": 70.96, "tpot_relative": 0.290362, "ttft_error_pct": 84.87, "ttft_relative": 0.151301 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 57.5, "tpot_relative": 0.319139, "ttft_error_pct": 7.49, "ttft_relative": 0.24056 }, "aisimulate": { "tpot_error_pct": 54.35, "tpot_relative": 0.312752, "ttft_error_pct": 39.07, "ttft_relative": 0.15842 }, "concurrency": 2, "measured": { "tpot_relative": 0.202625, "ttft_relative": 0.260025 }, "status": "success" }, { "aic": { "tpot_error_pct": 80.6, "tpot_relative": 0.403451, "ttft_error_pct": 81.02, "ttft_relative": 0.26742 }, "aisimulate": { "tpot_error_pct": 80.95, "tpot_relative": 0.396007, "ttft_error_pct": 87.39, "ttft_relative": 0.177678 }, "concurrency": 4, "measured": { "tpot_relative": 2.079236, "ttft_relative": 1.408823 }, "status": "success" } ], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 24.66, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.97, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 37.54, "tpot_shape_error_pct": null, "ttft_mape_pct": 4.7, "ttft_shape_error_pct": null }, "framework": "vllm", "id": "4d61695b0d8c39b6", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 24.66, "tpot_relative": 0.753415, "ttft_error_pct": 53.97, "ttft_relative": 0.460308 }, "aisimulate": { "tpot_error_pct": 37.54, "tpot_relative": 1.37541, "ttft_error_pct": 4.7, "ttft_relative": 0.953035 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 70.9, "tpot_shape_error_pct": 19.52, "ttft_mape_pct": 21.57, "ttft_shape_error_pct": 20.93 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 69.92, "tpot_shape_error_pct": 18.84, "ttft_mape_pct": 6.51, "ttft_shape_error_pct": 5.64 }, "framework": "vllm", "id": "98b35de55dfe189a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 46.4, "tpot_relative": 1.46398, "ttft_error_pct": 16.43, "ttft_relative": 1.164331 }, "aisimulate": { "tpot_error_pct": 46.3, "tpot_relative": 1.463027, "ttft_error_pct": 9.0, "ttft_relative": 0.909976 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 56.39, "tpot_relative": 1.815491, "ttft_error_pct": 23.2, "ttft_relative": 1.320926 }, "aisimulate": { "tpot_error_pct": 52.46, "tpot_relative": 1.769805, "ttft_error_pct": 10.73, "ttft_relative": 0.957126 }, "concurrency": 2, "measured": { "tpot_relative": 1.16084, "ttft_relative": 1.072182 }, "status": "success" }, { "aic": { "tpot_error_pct": 76.45, "tpot_relative": 2.488792, "ttft_error_pct": 26.08, "ttft_relative": 1.467936 }, "aisimulate": { "tpot_error_pct": 72.85, "tpot_relative": 2.437985, "ttft_error_pct": 7.6, "ttft_relative": 1.075823 }, "concurrency": 4, "measured": { "tpot_relative": 1.410458, "ttft_relative": 1.164289 }, "status": "success" }, { "aic": { "tpot_error_pct": 91.01, "tpot_relative": 3.506399, "ttft_error_pct": 10.61, "ttft_relative": 1.615706 }, "aisimulate": { "tpot_error_pct": 88.56, "tpot_relative": 3.46154, "ttft_error_pct": 7.84, "ttft_relative": 1.346155 }, "concurrency": 8, "measured": { "tpot_relative": 1.835748, "ttft_relative": 1.460736 }, "status": "success" }, { "aic": { "tpot_error_pct": 94.06, "tpot_relative": 5.03015, "ttft_error_pct": 0.23, "ttft_relative": 1.767432 }, "aisimulate": { "tpot_error_pct": 92.8, "tpot_relative": 4.997377, "ttft_error_pct": 2.67, "ttft_relative": 1.724164 }, "concurrency": 16, "measured": { "tpot_relative": 2.592, "ttft_relative": 1.77142 }, "status": "success" }, { "aic": { "tpot_error_pct": 73.06, "tpot_relative": 6.392399, "ttft_error_pct": 26.41, "ttft_relative": 1.923172 }, "aisimulate": { "tpot_error_pct": 73.99, "tpot_relative": 6.426787, "ttft_error_pct": 2.79, "ttft_relative": 2.540455 }, "concurrency": 32, "measured": { "tpot_relative": 3.693797, "ttft_relative": 2.613328 }, "status": "success" }, { "aic": { "tpot_error_pct": 58.9, "tpot_relative": 9.00917, "ttft_error_pct": 48.03, "ttft_relative": 2.084759 }, "aisimulate": { "tpot_error_pct": 62.5, "tpot_relative": 9.213602, "ttft_error_pct": 4.95, "ttft_relative": 4.209903 }, "concurrency": 64, "measured": { "tpot_relative": 5.669832, "ttft_relative": 4.011161 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 37, "tpot_mape_pct": 51.41, "tpot_shape_error_pct": 27.88, "ttft_mape_pct": 40.4, "ttft_shape_error_pct": 44.02 }, "aisimulate": { "coverage_pct": 100.0, "points": 37, "status_counts": { "failed": 0, "success": 37, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 46.98, "tpot_shape_error_pct": 26.12, "ttft_mape_pct": 25.33, "ttft_shape_error_pct": 41.87 }, "gpu": "b300", "precisions": [ "fp4" ], "rows": 37, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 21.61, "tpot_shape_error_pct": 24.77, "ttft_mape_pct": 28.05, "ttft_shape_error_pct": 40.27 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.02, "tpot_shape_error_pct": 23.14, "ttft_mape_pct": 46.13, "ttft_shape_error_pct": 6.25 }, "framework": "dynamo-vllm", "id": "25923af56467d45a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 2.55, "tpot_relative": 1.025519, "ttft_error_pct": 10.88, "ttft_relative": 1.108801 }, "aisimulate": { "tpot_error_pct": 4.94, "tpot_relative": 0.950595, "ttft_error_pct": 45.89, "ttft_relative": 0.541142 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.63, "tpot_relative": 1.364774, "ttft_error_pct": 18.59, "ttft_relative": 1.108801 }, "aisimulate": { "tpot_error_pct": 12.33, "tpot_relative": 1.250217, "ttft_error_pct": 51.45, "ttft_relative": 0.661168 }, "concurrency": 4, "measured": { "tpot_relative": 1.112938, "ttft_relative": 1.361915 }, "status": "success" }, { "aic": { "tpot_error_pct": 32.91, "tpot_relative": 1.679546, "ttft_error_pct": 36.0, "ttft_relative": 1.108801 }, "aisimulate": { "tpot_error_pct": 21.6, "tpot_relative": 1.536665, "ttft_error_pct": 42.46, "ttft_relative": 0.996893 }, "concurrency": 8, "measured": { "tpot_relative": 1.263699, "ttft_relative": 1.732616 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.34, "tpot_relative": 1.98854, "ttft_error_pct": 46.74, "ttft_relative": 1.108801 }, "aisimulate": { "tpot_error_pct": 17.22, "tpot_relative": 1.816345, "ttft_error_pct": 44.73, "ttft_relative": 1.150545 }, "concurrency": 16, "measured": { "tpot_relative": 1.549481, "ttft_relative": 2.081754 }, "status": "success" } ], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 72.34, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.85, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 54.71, "tpot_shape_error_pct": null, "ttft_mape_pct": 83.24, "ttft_shape_error_pct": null }, "framework": "dynamo-vllm", "id": "6df68390b6b1ed2a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 72.34, "tpot_relative": 1.723407, "ttft_error_pct": 93.85, "ttft_relative": 0.061537 }, "aisimulate": { "tpot_error_pct": 54.71, "tpot_relative": 1.547111, "ttft_error_pct": 83.24, "ttft_relative": 0.167604 }, "concurrency": 256, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 89.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.9, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 71.14, "tpot_shape_error_pct": null, "ttft_mape_pct": 81.83, "ttft_shape_error_pct": null }, "framework": "dynamo-vllm", "id": "0bfd3c476317d91d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 89.33, "tpot_relative": 1.893279, "ttft_error_pct": 91.9, "ttft_relative": 0.080984 }, "aisimulate": { "tpot_error_pct": 71.14, "tpot_relative": 1.711436, "ttft_error_pct": 81.83, "ttft_relative": 0.181744 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 74.86, "tpot_shape_error_pct": 8.75, "ttft_mape_pct": 57.23, "ttft_shape_error_pct": 60.38 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 57.48, "tpot_shape_error_pct": 8.56, "ttft_mape_pct": 45.91, "ttft_shape_error_pct": 46.73 }, "framework": "dynamo-vllm", "id": "e9510783cbbb540e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 72.3, "tpot_relative": 1.722985, "ttft_error_pct": 13.91, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 56.71, "tpot_relative": 1.567081, "ttft_error_pct": 11.41, "ttft_relative": 0.885871 }, "concurrency": 8, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 54.92, "tpot_relative": 2.03997, "ttft_error_pct": 35.45, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 40.89, "tpot_relative": 1.85525, "ttft_error_pct": 31.91, "ttft_relative": 0.908127 }, "concurrency": 16, "measured": { "tpot_relative": 1.316762, "ttft_relative": 1.333764 }, "status": "success" }, { "aic": { "tpot_error_pct": 60.08, "tpot_relative": 2.3529, "ttft_error_pct": 50.12, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 44.09, "tpot_relative": 2.118004, "ttft_error_pct": 32.81, "ttft_relative": 1.159538 }, "concurrency": 24, "measured": { "tpot_relative": 1.469872, "ttft_relative": 1.725867 }, "status": "success" }, { "aic": { "tpot_error_pct": 71.89, "tpot_relative": 2.705921, "ttft_error_pct": 63.65, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 53.91, "tpot_relative": 2.422907, "ttft_error_pct": 42.87, "ttft_relative": 1.353139 }, "concurrency": 32, "measured": { "tpot_relative": 1.574227, "ttft_relative": 2.368457 }, "status": "success" }, { "aic": { "tpot_error_pct": 89.86, "tpot_relative": 3.386223, "ttft_error_pct": 86.4, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 69.4, "tpot_relative": 3.021345, "ttft_error_pct": 72.74, "ttft_relative": 1.725945 }, "concurrency": 48, "measured": { "tpot_relative": 1.783509, "ttft_relative": 6.332073 }, "status": "success" }, { "aic": { "tpot_error_pct": 100.11, "tpot_relative": 3.589487, "ttft_error_pct": 93.85, "ttft_relative": 0.860939 }, "aisimulate": { "tpot_error_pct": 79.88, "tpot_relative": 3.226618, "ttft_error_pct": 83.71, "ttft_relative": 2.28003 }, "concurrency": 64, "measured": { "tpot_relative": 1.793733, "ttft_relative": 13.995345 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 40.96, "tpot_shape_error_pct": 1.88, "ttft_mape_pct": 33.63, "ttft_shape_error_pct": 3.85 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 39.48, "tpot_shape_error_pct": 4.18, "ttft_mape_pct": 7.29, "ttft_shape_error_pct": 4.25 }, "framework": "vllm", "id": "549155d33f94d660", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 42.29, "tpot_relative": 1.422935, "ttft_error_pct": 31.11, "ttft_relative": 1.311066 }, "aisimulate": { "tpot_error_pct": 42.46, "tpot_relative": 1.42459, "ttft_error_pct": 5.27, "ttft_relative": 0.947264 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 39.62, "tpot_relative": 1.588538, "ttft_error_pct": 36.15, "ttft_relative": 1.489737 }, "aisimulate": { "tpot_error_pct": 36.51, "tpot_relative": 1.553177, "ttft_error_pct": 9.3, "ttft_relative": 0.992421 }, "concurrency": 2, "measured": { "tpot_relative": 1.13777, "ttft_relative": 1.094204 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 10, "tpot_mape_pct": 61.75, "tpot_shape_error_pct": 58.09, "ttft_mape_pct": 37.65, "ttft_shape_error_pct": 45.82 }, "aisimulate": { "points": 10, "status_counts": { "failed": 0, "success": 10, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 61.28, "tpot_shape_error_pct": 57.58, "ttft_mape_pct": 18.67, "ttft_shape_error_pct": 88.29 }, "framework": "vllm", "id": "157dc2d4f2901b90", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 16.03, "tpot_relative": 1.16033, "ttft_error_pct": 30.92, "ttft_relative": 0.6908 }, "aisimulate": { "tpot_error_pct": 16.04, "tpot_relative": 1.160427, "ttft_error_pct": 53.57, "ttft_relative": 0.464342 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.42, "tpot_relative": 1.191556, "ttft_error_pct": 8.82, "ttft_relative": 0.78619 }, "aisimulate": { "tpot_error_pct": 8.62, "tpot_relative": 1.172111, "ttft_error_pct": 43.79, "ttft_relative": 0.484709 }, "concurrency": 2, "measured": { "tpot_relative": 1.0791, "ttft_relative": 0.862281 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.53, "tpot_relative": 1.460287, "ttft_error_pct": 17.81, "ttft_relative": 0.873982 }, "aisimulate": { "tpot_error_pct": 19.51, "tpot_relative": 1.436081, "ttft_error_pct": 26.7, "ttft_relative": 0.543771 }, "concurrency": 4, "measured": { "tpot_relative": 1.201615, "ttft_relative": 0.741879 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.11, "tpot_relative": 1.775335, "ttft_error_pct": 4.32, "ttft_relative": 0.961461 }, "aisimulate": { "tpot_error_pct": 5.27, "tpot_relative": 1.753903, "ttft_error_pct": 32.85, "ttft_relative": 0.674772 }, "concurrency": 8, "measured": { "tpot_relative": 1.851513, "ttft_relative": 1.004874 }, "status": "success" }, { "aic": { "tpot_error_pct": 509.13, "tpot_relative": 11.217997, "ttft_error_pct": 8.54, "ttft_relative": 1.050024 }, "aisimulate": { "tpot_error_pct": 507.55, "tpot_relative": 11.188789, "ttft_error_pct": 5.81, "ttft_relative": 1.081292 }, "concurrency": 16, "measured": { "tpot_relative": 1.841635, "ttft_relative": 1.14802 }, "status": "success" }, { "aic": { "tpot_error_pct": 23.59, "tpot_relative": 3.078315, "ttft_error_pct": 24.16, "ttft_relative": 1.144761 }, "aisimulate": { "tpot_error_pct": 28.16, "tpot_relative": 3.192069, "ttft_error_pct": 9.16, "ttft_relative": 1.37123 }, "concurrency": 32, "measured": { "tpot_relative": 2.490762, "ttft_relative": 1.509426 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.06, "tpot_relative": 4.057549, "ttft_error_pct": 46.93, "ttft_relative": 1.244136 }, "aisimulate": { "tpot_error_pct": 13.16, "tpot_relative": 4.134183, "ttft_error_pct": 4.09, "ttft_relative": 2.248356 }, "concurrency": 64, "measured": { "tpot_relative": 3.653532, "ttft_relative": 2.344192 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.63, "tpot_relative": 5.611217, "ttft_error_pct": 66.3, "ttft_relative": 1.353929 }, "aisimulate": { "tpot_error_pct": 0.74, "tpot_relative": 5.72036, "ttft_error_pct": 0.75, "ttft_relative": 3.987548 }, "concurrency": 128, "measured": { "tpot_relative": 5.762961, "ttft_relative": 4.017617 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.14, "tpot_relative": 8.599635, "ttft_error_pct": 79.74, "ttft_relative": 1.487895 }, "aisimulate": { "tpot_error_pct": 5.67, "tpot_relative": 8.830994, "ttft_error_pct": 3.18, "ttft_relative": 7.576711 }, "concurrency": 256, "measured": { "tpot_relative": 9.361843, "ttft_relative": 7.342858 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.87, "tpot_relative": 14.765492, "ttft_error_pct": 89.02, "ttft_relative": 1.577059 }, "aisimulate": { "tpot_error_pct": 8.06, "tpot_relative": 15.230629, "ttft_error_pct": 6.84, "ttft_relative": 15.34314 }, "concurrency": 512, "measured": { "tpot_relative": 16.566426, "ttft_relative": 14.360828 }, "status": "success" } ], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 54.63, "tpot_shape_error_pct": 2.4, "ttft_mape_pct": 28.97, "ttft_shape_error_pct": 7.39 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 53.1, "tpot_shape_error_pct": 0.38, "ttft_mape_pct": 12.19, "ttft_shape_error_pct": 1.05 }, "framework": "vllm", "id": "716ecd2709264f0a", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 52.79, "tpot_relative": 1.527907, "ttft_error_pct": 24.37, "ttft_relative": 1.243719 }, "aisimulate": { "tpot_error_pct": 52.8, "tpot_relative": 1.528033, "ttft_error_pct": 11.73, "ttft_relative": 0.882697 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 56.46, "tpot_relative": 1.664183, "ttft_error_pct": 33.57, "ttft_relative": 1.413286 }, "aisimulate": { "tpot_error_pct": 53.39, "tpot_relative": 1.631491, "ttft_error_pct": 12.66, "ttft_relative": 0.924209 }, "concurrency": 2, "measured": { "tpot_relative": 1.063629, "ttft_relative": 1.058121 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 21.82, "tpot_shape_error_pct": 14.37, "ttft_mape_pct": 22.71, "ttft_shape_error_pct": 25.53 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 25.22, "tpot_shape_error_pct": 2.59, "ttft_mape_pct": 11.42, "ttft_shape_error_pct": 0.33 }, "framework": "vllm", "id": "c9340cab98c7bedd", "parallelism": { "attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 15.77, "tpot_relative": 0.842306, "ttft_error_pct": 11.39, "ttft_relative": 0.886069 }, "aisimulate": { "tpot_error_pct": 26.86, "tpot_relative": 1.268613, "ttft_error_pct": 11.57, "ttft_relative": 0.884311 }, "concurrency": 64, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.88, "tpot_relative": 1.084718, "ttft_error_pct": 34.02, "ttft_relative": 0.963846 }, "aisimulate": { "tpot_error_pct": 23.58, "tpot_relative": 1.858614, "ttft_error_pct": 11.28, "ttft_relative": 1.29605 }, "concurrency": 128, "measured": { "tpot_relative": 1.503986, "ttft_relative": 1.460768 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 47.04, "tpot_shape_error_pct": 15.36, "ttft_mape_pct": 20.62, "ttft_shape_error_pct": 29.16 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 49.55, "tpot_shape_error_pct": 3.17, "ttft_mape_pct": 7.04, "ttft_shape_error_pct": 5.16 }, "framework": "vllm", "id": "5b9fdb9172dc6e8e", "parallelism": { "attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1 }, "points": [ { "aic": { "tpot_error_pct": 42.64, "tpot_relative": 0.573622, "ttft_error_pct": 7.07, "ttft_relative": 0.929322 }, "aisimulate": { "tpot_error_pct": 51.95, "tpot_relative": 1.519522, "ttft_error_pct": 4.58, "ttft_relative": 0.954236 }, "concurrency": 128, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 51.45, "tpot_relative": 0.802437, "ttft_error_pct": 34.16, "ttft_relative": 1.01037 }, "aisimulate": { "tpot_error_pct": 47.14, "tpot_relative": 2.431946, "ttft_error_pct": 9.5, "ttft_relative": 1.388914 }, "concurrency": 256, "measured": { "tpot_relative": 1.652822, "ttft_relative": 1.534654 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 36.92, "tpot_shape_error_pct": 12.97, "ttft_mape_pct": 37.85, "ttft_shape_error_pct": 47.9 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 37.73, "tpot_shape_error_pct": 10.74, "ttft_mape_pct": 7.05, "ttft_shape_error_pct": 12.11 }, "framework": "vllm", "id": "dde7884dd68cb4d1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 48.51, "tpot_relative": 1.485056, "ttft_error_pct": 26.74, "ttft_relative": 1.26737 }, "aisimulate": { "tpot_error_pct": 45.55, "tpot_relative": 1.455485, "ttft_error_pct": 10.15, "ttft_relative": 0.89851 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 64.59, "tpot_relative": 2.065309, "ttft_error_pct": 17.17, "ttft_relative": 1.395277 }, "aisimulate": { "tpot_error_pct": 62.27, "tpot_relative": 2.036227, "ttft_error_pct": 5.67, "ttft_relative": 1.12326 }, "concurrency": 8, "measured": { "tpot_relative": 1.254837, "ttft_relative": 1.190782 }, "status": "success" }, { "aic": { "tpot_error_pct": 49.68, "tpot_relative": 2.769166, "ttft_error_pct": 0.42, "ttft_relative": 1.526241 }, "aisimulate": { "tpot_error_pct": 48.35, "tpot_relative": 2.744585, "ttft_error_pct": 6.56, "ttft_relative": 1.432193 }, "concurrency": 16, "measured": { "tpot_relative": 1.850077, "ttft_relative": 1.53266 }, "status": "success" }, { "aic": { "tpot_error_pct": 39.09, "tpot_relative": 3.619797, "ttft_error_pct": 23.9, "ttft_relative": 1.662122 }, "aisimulate": { "tpot_error_pct": 39.24, "tpot_relative": 3.623737, "ttft_error_pct": 3.36, "ttft_relative": 2.110593 }, "concurrency": 32, "measured": { "tpot_relative": 2.602538, "ttft_relative": 2.184011 }, "status": "success" }, { "aic": { "tpot_error_pct": 31.36, "tpot_relative": 5.242377, "ttft_error_pct": 49.31, "ttft_relative": 1.803601 }, "aisimulate": { "tpot_error_pct": 33.54, "tpot_relative": 5.3294, "ttft_error_pct": 1.82, "ttft_relative": 3.493523 }, "concurrency": 64, "measured": { "tpot_relative": 3.990844, "ttft_relative": 3.558333 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.56, "tpot_relative": 7.256701, "ttft_error_pct": 67.05, "ttft_relative": 1.960717 }, "aisimulate": { "tpot_error_pct": 21.51, "tpot_relative": 7.500609, "ttft_error_pct": 7.56, "ttft_relative": 6.399306 }, "concurrency": 128, "measured": { "tpot_relative": 6.172786, "ttft_relative": 5.949777 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.7, "tpot_relative": 10.952061, "ttft_error_pct": 80.34, "ttft_relative": 2.153913 }, "aisimulate": { "tpot_error_pct": 13.64, "tpot_relative": 11.556432, "ttft_error_pct": 14.27, "ttft_relative": 12.517063 }, "concurrency": 256, "measured": { "tpot_relative": 10.1694, "ttft_relative": 10.953745 }, "status": "success" } ], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "fp4" ], "rows": 70 } ] }, { "aic": { "points": 104, "tpot_mape_pct": 21.53, "tpot_shape_error_pct": 4.2, "ttft_mape_pct": 47.76, "ttft_shape_error_pct": 31.32 }, "aisimulate": { "coverage_pct": 100.0, "points": 104, "status_counts": { "failed": 0, "success": 104, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 21.13, "tpot_shape_error_pct": 5.03, "ttft_mape_pct": 78.48, "ttft_shape_error_pct": 48.9 }, "frameworks": [ "sglang", "trtllm" ], "gpu_skus": [ "b200", "b300", "h100" ], "hf_model_paths": [ "Qwen/Qwen3.5-397B-A17B" ], "model": "Qwen-3.5-397B-A17B", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 104, "workloads": [ { "aic": { "points": 40, "tpot_mape_pct": 14.75, "tpot_shape_error_pct": 4.94, "ttft_mape_pct": 64.98, "ttft_shape_error_pct": 16.15 }, "aisimulate": { "coverage_pct": 100.0, "points": 40, "status_counts": { "failed": 0, "success": 40, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.82, "tpot_shape_error_pct": 4.97, "ttft_mape_pct": 80.87, "ttft_shape_error_pct": 16.59 }, "gpu_skus": [ "b200", "b300", "h100" ], "gpus": [ { "aic": { "points": 7, "tpot_mape_pct": 34.36, "tpot_shape_error_pct": 5.0, "ttft_mape_pct": 73.71, "ttft_shape_error_pct": 7.63 }, "aisimulate": { "coverage_pct": 100.0, "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 34.26, "tpot_shape_error_pct": 5.1, "ttft_mape_pct": 85.61, "ttft_shape_error_pct": 8.48 }, "gpu": "b200", "precisions": [ "bf16", "fp4" ], "rows": 7, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 7.3, "tpot_shape_error_pct": 5.49, "ttft_mape_pct": 74.03, "ttft_shape_error_pct": 5.57 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.36, "tpot_shape_error_pct": 5.74, "ttft_mape_pct": 86.86, "ttft_shape_error_pct": 7.28 }, "framework": "sglang", "id": "626dca35e08973d3", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 3.71, "tpot_relative": 0.962871, "ttft_error_pct": 73.19, "ttft_relative": 0.268125 }, "aisimulate": { "tpot_error_pct": 3.19, "tpot_relative": 0.968132, "ttft_error_pct": 86.26, "ttft_relative": 0.137415 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 7.23, "tpot_relative": 1.228753, "ttft_error_pct": 74.15, "ttft_relative": 0.29486 }, "aisimulate": { "tpot_error_pct": 6.75, "tpot_relative": 1.235176, "ttft_error_pct": 86.62, "ttft_relative": 0.152612 }, "concurrency": 8, "measured": { "tpot_relative": 1.324522, "ttft_relative": 1.14059 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.79, "tpot_relative": 1.499869, "ttft_error_pct": 76.7, "ttft_relative": 0.348328 }, "aisimulate": { "tpot_error_pct": 9.76, "tpot_relative": 1.517179, "ttft_error_pct": 88.55, "ttft_relative": 0.171182 }, "concurrency": 16, "measured": { "tpot_relative": 1.681274, "ttft_relative": 1.494764 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.67, "tpot_relative": 1.900672, "ttft_error_pct": 72.32, "ttft_relative": 0.456488 }, "aisimulate": { "tpot_error_pct": 10.92, "tpot_relative": 1.938758, "ttft_error_pct": 87.12, "ttft_relative": 0.212395 }, "concurrency": 32, "measured": { "tpot_relative": 2.176436, "ttft_relative": 1.648881 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.11, "tpot_relative": 2.752911, "ttft_error_pct": 73.82, "ttft_relative": 0.538405 }, "aisimulate": { "tpot_error_pct": 1.18, "tpot_relative": 2.84545, "ttft_error_pct": 85.77, "ttft_relative": 0.292717 }, "concurrency": 64, "measured": { "tpot_relative": 2.812213, "ttft_relative": 2.056767 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 101.99, "tpot_shape_error_pct": 3.03, "ttft_mape_pct": 72.9, "ttft_shape_error_pct": 15.84 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 104.01, "tpot_shape_error_pct": 2.52, "ttft_mape_pct": 82.48, "ttft_shape_error_pct": 13.28 }, "framework": "trtllm", "id": "c0b1b6a359759a4e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 105.1, "tpot_relative": 2.050951, "ttft_error_pct": 74.89, "ttft_relative": 0.251078 }, "aisimulate": { "tpot_error_pct": 106.61, "tpot_relative": 2.066145, "ttft_error_pct": 83.57, "ttft_relative": 0.164319 }, "concurrency": 8, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 98.89, "tpot_relative": 2.40957, "ttft_error_pct": 70.91, "ttft_relative": 0.296769 }, "aisimulate": { "tpot_error_pct": 101.41, "tpot_relative": 2.440131, "ttft_error_pct": 81.39, "ttft_relative": 0.189928 }, "concurrency": 16, "measured": { "tpot_relative": 1.211539, "ttft_relative": 1.020328 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 29, "tpot_mape_pct": 9.4, "tpot_shape_error_pct": 5.23, "ttft_mape_pct": 61.43, "ttft_shape_error_pct": 18.11 }, "aisimulate": { "coverage_pct": 100.0, "points": 29, "status_counts": { "failed": 0, "success": 29, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 9.56, "tpot_shape_error_pct": 5.18, "ttft_mape_pct": 78.83, "ttft_shape_error_pct": 18.72 }, "gpu": "b300", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 29, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 4.7, "tpot_shape_error_pct": 2.93, "ttft_mape_pct": 62.28, "ttft_shape_error_pct": 13.12 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 4.6, "tpot_shape_error_pct": 3.2, "ttft_mape_pct": 81.01, "ttft_shape_error_pct": 14.0 }, "framework": "sglang", "id": "62c57879d9e0089d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 4.42, "tpot_relative": 0.955839, "ttft_error_pct": 64.85, "ttft_relative": 0.351456 }, "aisimulate": { "tpot_error_pct": 3.9, "tpot_relative": 0.961003, "ttft_error_pct": 82.08, "ttft_relative": 0.179212 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.93, "tpot_relative": 1.229742, "ttft_error_pct": 50.32, "ttft_relative": 0.386355 }, "aisimulate": { "tpot_error_pct": 3.4, "tpot_relative": 1.236547, "ttft_error_pct": 74.4, "ttft_relative": 0.199052 }, "concurrency": 8, "measured": { "tpot_relative": 1.280009, "ttft_relative": 0.777695 }, "status": "success" }, { "aic": { "tpot_error_pct": 5.97, "tpot_relative": 1.534555, "ttft_error_pct": 65.75, "ttft_relative": 0.456619 }, "aisimulate": { "tpot_error_pct": 4.91, "tpot_relative": 1.5519, "ttft_error_pct": 83.26, "ttft_relative": 0.223179 }, "concurrency": 16, "measured": { "tpot_relative": 1.632008, "ttft_relative": 1.333299 }, "status": "success" }, { "aic": { "tpot_error_pct": 8.51, "tpot_relative": 1.937803, "ttft_error_pct": 63.72, "ttft_relative": 0.598995 }, "aisimulate": { "tpot_error_pct": 6.78, "tpot_relative": 1.974438, "ttft_error_pct": 83.24, "ttft_relative": 0.276675 }, "concurrency": 32, "measured": { "tpot_relative": 2.118107, "ttft_relative": 1.65087 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.66, "tpot_relative": 2.791592, "ttft_error_pct": 66.73, "ttft_relative": 0.706417 }, "aisimulate": { "tpot_error_pct": 4.0, "tpot_relative": 2.884264, "ttft_error_pct": 82.06, "ttft_relative": 0.380838 }, "concurrency": 64, "measured": { "tpot_relative": 2.773401, "ttft_relative": 2.123372 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 5.36, "tpot_shape_error_pct": 3.81, "ttft_mape_pct": 52.98, "ttft_shape_error_pct": 29.23 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 6.27, "tpot_shape_error_pct": 4.24, "ttft_mape_pct": 76.71, "ttft_shape_error_pct": 32.01 }, "framework": "sglang", "id": "2dde0967497d0cf1", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 3.88, "tpot_relative": 1.038818, "ttft_error_pct": 41.38, "ttft_relative": 0.586192 }, "aisimulate": { "tpot_error_pct": 4.52, "tpot_relative": 1.045194, "ttft_error_pct": 70.12, "ttft_relative": 0.298816 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.67, "tpot_relative": 1.356378, "ttft_error_pct": 36.11, "ttft_relative": 0.644508 }, "aisimulate": { "tpot_error_pct": 7.32, "tpot_relative": 1.364612, "ttft_error_pct": 67.47, "ttft_relative": 0.328111 }, "concurrency": 8, "measured": { "tpot_relative": 1.271574, "ttft_relative": 1.008765 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.55, "tpot_relative": 1.809966, "ttft_error_pct": 59.39, "ttft_relative": 0.761452 }, "aisimulate": { "tpot_error_pct": 4.68, "tpot_relative": 1.829662, "ttft_error_pct": 80.54, "ttft_relative": 0.364819 }, "concurrency": 16, "measured": { "tpot_relative": 1.747938, "ttft_relative": 1.875134 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.98, "tpot_relative": 2.376702, "ttft_error_pct": 62.87, "ttft_relative": 0.997932 }, "aisimulate": { "tpot_error_pct": 0.04, "tpot_relative": 2.425746, "ttft_error_pct": 83.39, "ttft_relative": 0.44649 }, "concurrency": 32, "measured": { "tpot_relative": 2.424672, "ttft_relative": 2.687536 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.74, "tpot_relative": 3.577173, "ttft_error_pct": 65.14, "ttft_relative": 1.175339 }, "aisimulate": { "tpot_error_pct": 14.8, "tpot_relative": 3.70835, "ttft_error_pct": 82.04, "ttft_relative": 0.605499 }, "concurrency": 64, "measured": { "tpot_relative": 3.23033, "ttft_relative": 3.371498 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 28.46, "tpot_shape_error_pct": 3.23, "ttft_mape_pct": 74.42, "ttft_shape_error_pct": 23.61 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 27.03, "tpot_shape_error_pct": 3.84, "ttft_mape_pct": 86.22, "ttft_shape_error_pct": 16.8 }, "framework": "sglang", "id": "2c525cf6b185298f", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 27.51, "tpot_relative": 0.724926, "ttft_error_pct": 69.18, "ttft_relative": 0.30821 }, "aisimulate": { "tpot_error_pct": 27.13, "tpot_relative": 0.728685, "ttft_error_pct": 84.59, "ttft_relative": 0.154113 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 24.64, "tpot_relative": 0.916012, "ttft_error_pct": 66.71, "ttft_relative": 0.338837 }, "aisimulate": { "tpot_error_pct": 24.36, "tpot_relative": 0.919325, "ttft_error_pct": 83.02, "ttft_relative": 0.172851 }, "concurrency": 8, "measured": { "tpot_relative": 1.215448, "ttft_relative": 1.017737 }, "status": "success" }, { "aic": { "tpot_error_pct": 30.5, "tpot_relative": 1.103389, "ttft_error_pct": 76.6, "ttft_relative": 0.400124 }, "aisimulate": { "tpot_error_pct": 29.86, "tpot_relative": 1.113562, "ttft_error_pct": 88.59, "ttft_relative": 0.195007 }, "concurrency": 16, "measured": { "tpot_relative": 1.587685, "ttft_relative": 1.709716 }, "status": "success" }, { "aic": { "tpot_error_pct": 32.27, "tpot_relative": 1.376142, "ttft_error_pct": 74.33, "ttft_relative": 0.524058 }, "aisimulate": { "tpot_error_pct": 31.09, "tpot_relative": 1.399933, "ttft_error_pct": 88.02, "ttft_relative": 0.244535 }, "concurrency": 32, "measured": { "tpot_relative": 2.031664, "ttft_relative": 2.041528 }, "status": "success" }, { "aic": { "tpot_error_pct": 28.48, "tpot_relative": 1.871872, "ttft_error_pct": 76.72, "ttft_relative": 0.617596 }, "aisimulate": { "tpot_error_pct": 26.41, "tpot_relative": 1.925853, "ttft_error_pct": 87.14, "ttft_relative": 0.341036 }, "concurrency": 64, "measured": { "tpot_relative": 2.617177, "ttft_relative": 2.652819 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.39, "tpot_relative": 2.520507, "ttft_error_pct": 82.98, "ttft_relative": 0.643599 }, "aisimulate": { "tpot_error_pct": 23.33, "tpot_relative": 2.661672, "ttft_error_pct": 85.97, "ttft_relative": 0.530524 }, "concurrency": 128, "measured": { "tpot_relative": 3.471403, "ttft_relative": 3.78045 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 4.82, "tpot_shape_error_pct": 7.88, "ttft_mape_pct": 65.88, "ttft_shape_error_pct": 17.0 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 5.68, "tpot_shape_error_pct": 9.87, "ttft_mape_pct": 81.48, "ttft_shape_error_pct": 13.5 }, "framework": "sglang", "id": "bbdf1d61773378d0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 7.83, "tpot_relative": 0.921734, "ttft_error_pct": 61.69, "ttft_relative": 0.383053 }, "aisimulate": { "tpot_error_pct": 7.23, "tpot_relative": 0.927711, "ttft_error_pct": 80.44, "ttft_relative": 0.195649 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.41, "tpot_relative": 1.164776, "ttft_error_pct": 57.95, "ttft_relative": 0.421258 }, "aisimulate": { "tpot_error_pct": 0.89, "tpot_relative": 1.170845, "ttft_error_pct": 78.32, "ttft_relative": 0.21719 }, "concurrency": 8, "measured": { "tpot_relative": 1.18139, "ttft_relative": 1.001877 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.1, "tpot_relative": 1.424921, "ttft_error_pct": 68.08, "ttft_relative": 0.497825 }, "aisimulate": { "tpot_error_pct": 3.05, "tpot_relative": 1.440521, "ttft_error_pct": 84.38, "ttft_relative": 0.243686 }, "concurrency": 16, "measured": { "tpot_relative": 1.485901, "ttft_relative": 1.559765 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.45, "tpot_relative": 1.786992, "ttft_error_pct": 66.35, "ttft_relative": 0.652922 }, "aisimulate": { "tpot_error_pct": 4.66, "tpot_relative": 1.821289, "ttft_error_pct": 84.39, "ttft_relative": 0.303015 }, "concurrency": 32, "measured": { "tpot_relative": 1.910271, "ttft_relative": 1.940607 }, "status": "success" }, { "aic": { "tpot_error_pct": 2.28, "tpot_relative": 2.496034, "ttft_error_pct": 67.32, "ttft_relative": 0.771107 }, "aisimulate": { "tpot_error_pct": 5.4, "tpot_relative": 2.572282, "ttft_error_pct": 82.27, "ttft_relative": 0.418388 }, "concurrency": 64, "measured": { "tpot_relative": 2.440398, "ttft_relative": 2.359865 }, "status": "success" }, { "aic": { "tpot_error_pct": 6.85, "tpot_relative": 3.403073, "ttft_error_pct": 73.84, "ttft_relative": 0.805836 }, "aisimulate": { "tpot_error_pct": 12.85, "tpot_relative": 3.594097, "ttft_error_pct": 79.07, "ttft_relative": 0.644828 }, "concurrency": 128, "measured": { "tpot_relative": 3.184843, "ttft_relative": 3.080456 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 3.23, "tpot_shape_error_pct": 7.18, "ttft_mape_pct": 51.93, "ttft_shape_error_pct": 10.35 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 3.81, "tpot_shape_error_pct": 4.35, "ttft_mape_pct": 70.18, "ttft_shape_error_pct": 18.97 }, "framework": "sglang", "id": "f6061c407ba6b6c3", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 6.48, "tpot_relative": 1.064806, "ttft_error_pct": 49.66, "ttft_relative": 0.503428 }, "aisimulate": { "tpot_error_pct": 7.02, "tpot_relative": 1.070194, "ttft_error_pct": 73.88, "ttft_relative": 0.261237 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.58, "tpot_relative": 1.260173, "ttft_error_pct": 47.98, "ttft_relative": 0.553869 }, "aisimulate": { "tpot_error_pct": 5.39, "tpot_relative": 1.269934, "ttft_error_pct": 72.64, "ttft_relative": 0.291324 }, "concurrency": 8, "measured": { "tpot_relative": 1.205017, "ttft_relative": 1.064785 }, "status": "success" }, { "aic": { "tpot_error_pct": 0.24, "tpot_relative": 1.55875, "ttft_error_pct": 46.62, "ttft_relative": 0.655182 }, "aisimulate": { "tpot_error_pct": 1.01, "tpot_relative": 1.578289, "ttft_error_pct": 73.4, "ttft_relative": 0.326444 }, "concurrency": 16, "measured": { "tpot_relative": 1.562534, "ttft_relative": 1.227373 }, "status": "success" }, { "aic": { "tpot_error_pct": 4.82, "tpot_relative": 1.971693, "ttft_error_pct": 48.54, "ttft_relative": 0.860812 }, "aisimulate": { "tpot_error_pct": 2.75, "tpot_relative": 2.014593, "ttft_error_pct": 75.79, "ttft_relative": 0.405008 }, "concurrency": 32, "measured": { "tpot_relative": 2.071574, "ttft_relative": 1.672774 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.45, "tpot_relative": 2.721839, "ttft_error_pct": 47.8, "ttft_relative": 1.019039 }, "aisimulate": { "tpot_error_pct": 1.95, "tpot_relative": 2.815659, "ttft_error_pct": 71.44, "ttft_relative": 0.557577 }, "concurrency": 64, "measured": { "tpot_relative": 2.761934, "ttft_relative": 1.952144 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.23, "tpot_relative": 3.565579, "ttft_error_pct": 53.73, "ttft_relative": 1.067261 }, "aisimulate": { "tpot_error_pct": 2.45, "tpot_relative": 3.775134, "ttft_error_pct": 62.83, "ttft_relative": 0.85736 }, "concurrency": 128, "measured": { "tpot_relative": 3.68468, "ttft_relative": 2.306706 }, "status": "success" }, { "aic": { "tpot_error_pct": 1.79, "tpot_relative": 4.860219, "ttft_error_pct": 69.15, "ttft_relative": 1.160547 }, "aisimulate": { "tpot_error_pct": 6.13, "tpot_relative": 5.252063, "ttft_error_pct": 61.25, "ttft_relative": 1.457716 }, "concurrency": 256, "measured": { "tpot_relative": 4.948638, "ttft_relative": 3.76225 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 4, "tpot_mape_pct": 19.22, "tpot_shape_error_pct": 2.45, "ttft_mape_pct": 75.43, "ttft_shape_error_pct": 14.7 }, "aisimulate": { "coverage_pct": 100.0, "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.87, "tpot_shape_error_pct": 3.05, "ttft_mape_pct": 87.38, "ttft_shape_error_pct": 13.01 }, "gpu": "h100", "precisions": [ "fp8" ], "rows": 4, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 19.22, "tpot_shape_error_pct": 2.45, "ttft_mape_pct": 75.43, "ttft_shape_error_pct": 14.7 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.87, "tpot_shape_error_pct": 3.05, "ttft_mape_pct": 87.38, "ttft_shape_error_pct": 13.01 }, "framework": "sglang", "id": "07f86e5ee2a642ea", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 20.68, "tpot_relative": 0.793201, "ttft_error_pct": 75.34, "ttft_relative": 0.246594 }, "aisimulate": { "tpot_error_pct": 20.69, "tpot_relative": 0.793135, "ttft_error_pct": 87.43, "ttft_relative": 0.125675 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.46, "tpot_relative": 0.863759, "ttft_error_pct": 73.97, "ttft_relative": 0.261312 }, "aisimulate": { "tpot_error_pct": 17.58, "tpot_relative": 0.862503, "ttft_error_pct": 86.94, "ttft_relative": 0.131085 }, "concurrency": 2, "measured": { "tpot_relative": 1.046517, "ttft_relative": 1.004044 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.1, "tpot_relative": 1.015005, "ttft_error_pct": 71.45, "ttft_relative": 0.274975 }, "aisimulate": { "tpot_error_pct": 19.28, "tpot_relative": 1.025335, "ttft_error_pct": 85.35, "ttft_relative": 0.141045 }, "concurrency": 4, "measured": { "tpot_relative": 1.270272, "ttft_relative": 0.963001 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.64, "tpot_relative": 1.277941, "ttft_error_pct": 80.96, "ttft_relative": 0.302362 }, "aisimulate": { "tpot_error_pct": 17.94, "tpot_relative": 1.288992, "ttft_error_pct": 89.77, "ttft_relative": 0.162392 }, "concurrency": 8, "measured": { "tpot_relative": 1.570719, "ttft_relative": 1.587663 }, "status": "success" } ], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "1024:1024", "label": "1k1k", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 40 }, { "aic": { "points": 64, "tpot_mape_pct": 25.76, "tpot_shape_error_pct": 3.73, "ttft_mape_pct": 37.01, "ttft_shape_error_pct": 41.03 }, "aisimulate": { "coverage_pct": 100.0, "points": 64, "status_counts": { "failed": 0, "success": 64, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 25.08, "tpot_shape_error_pct": 5.07, "ttft_mape_pct": 76.98, "ttft_shape_error_pct": 69.58 }, "gpu_skus": [ "b200", "b300", "h100" ], "gpus": [ { "aic": { "points": 31, "tpot_mape_pct": 37.74, "tpot_shape_error_pct": 2.56, "ttft_mape_pct": 44.3, "ttft_shape_error_pct": 45.23 }, "aisimulate": { "coverage_pct": 100.0, "points": 31, "status_counts": { "failed": 0, "success": 31, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 35.8, "tpot_shape_error_pct": 5.47, "ttft_mape_pct": 105.34, "ttft_shape_error_pct": 130.72 }, "gpu": "b200", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 31, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 13.19, "tpot_shape_error_pct": 1.07, "ttft_mape_pct": 34.92, "ttft_shape_error_pct": 28.97 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 13.87, "tpot_shape_error_pct": 1.74, "ttft_mape_pct": 59.26, "ttft_shape_error_pct": 8.0 }, "framework": "sglang", "id": "7e3c467c32b7a577", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 13.75, "tpot_relative": 1.137523, "ttft_error_pct": 15.29, "ttft_relative": 0.847144 }, "aisimulate": { "tpot_error_pct": 15.48, "tpot_relative": 1.154791, "ttft_error_pct": 57.0, "ttft_relative": 0.429996 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.59, "tpot_relative": 1.496845, "ttft_error_pct": 29.57, "ttft_relative": 0.931336 }, "aisimulate": { "tpot_error_pct": 13.36, "tpot_relative": 1.493797, "ttft_error_pct": 61.3, "ttft_relative": 0.511784 }, "concurrency": 8, "measured": { "tpot_relative": 1.317713, "ttft_relative": 1.322375 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.77, "tpot_relative": 2.023701, "ttft_error_pct": 36.17, "ttft_relative": 1.103774 }, "aisimulate": { "tpot_error_pct": 14.48, "tpot_relative": 2.018635, "ttft_error_pct": 62.13, "ttft_relative": 0.654871 }, "concurrency": 16, "measured": { "tpot_relative": 1.763323, "ttft_relative": 1.729326 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.39, "tpot_relative": 2.685879, "ttft_error_pct": 40.64, "ttft_relative": 1.44087 }, "aisimulate": { "tpot_error_pct": 10.96, "tpot_relative": 2.699835, "ttft_error_pct": 60.1, "ttft_relative": 0.968477 }, "concurrency": 32, "measured": { "tpot_relative": 2.433156, "ttft_relative": 2.427433 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.44, "tpot_relative": 4.037475, "ttft_error_pct": 52.93, "ttft_relative": 1.677395 }, "aisimulate": { "tpot_error_pct": 15.05, "tpot_relative": 4.095041, "ttft_error_pct": 55.78, "ttft_relative": 1.575723 }, "concurrency": 64, "measured": { "tpot_relative": 3.55923, "ttft_relative": 3.563342 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 26.03, "tpot_shape_error_pct": null, "ttft_mape_pct": 47.05, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 28.03, "tpot_shape_error_pct": null, "ttft_mape_pct": 25.6, "ttft_shape_error_pct": null }, "framework": "sglang", "id": "06c58b407434b1df", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 26.03, "tpot_relative": 1.260276, "ttft_error_pct": 47.05, "ttft_relative": 1.470487 }, "aisimulate": { "tpot_error_pct": 28.03, "tpot_relative": 1.280276, "ttft_error_pct": 25.6, "ttft_relative": 0.744032 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 26.87, "tpot_shape_error_pct": 2.28, "ttft_mape_pct": 30.98, "ttft_shape_error_pct": 49.26 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 27.95, "tpot_shape_error_pct": 2.04, "ttft_mape_pct": 44.6, "ttft_shape_error_pct": 24.83 }, "framework": "sglang", "id": "533d8b7c8078c2ee", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 25.62, "tpot_relative": 1.256242, "ttft_error_pct": 39.3, "ttft_relative": 1.392978 }, "aisimulate": { "tpot_error_pct": 28.33, "tpot_relative": 1.283276, "ttft_error_pct": 30.15, "ttft_relative": 0.698525 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 27.05, "tpot_relative": 1.597471, "ttft_error_pct": 3.04, "ttft_relative": 1.530266 }, "aisimulate": { "tpot_error_pct": 27.5, "tpot_relative": 1.603184, "ttft_error_pct": 47.36, "ttft_relative": 0.830756 }, "concurrency": 8, "measured": { "tpot_relative": 1.257361, "ttft_relative": 1.578264 }, "status": "success" }, { "aic": { "tpot_error_pct": 33.89, "tpot_relative": 2.333986, "ttft_error_pct": 9.56, "ttft_relative": 1.811718 }, "aisimulate": { "tpot_error_pct": 33.74, "tpot_relative": 2.331261, "ttft_error_pct": 46.92, "ttft_relative": 1.063327 }, "concurrency": 16, "measured": { "tpot_relative": 1.743156, "ttft_relative": 2.003325 }, "status": "success" }, { "aic": { "tpot_error_pct": 25.24, "tpot_relative": 3.189788, "ttft_error_pct": 23.56, "ttft_relative": 2.364336 }, "aisimulate": { "tpot_error_pct": 25.64, "tpot_relative": 3.200012, "ttft_error_pct": 49.16, "ttft_relative": 1.572285 }, "concurrency": 32, "measured": { "tpot_relative": 2.546922, "ttft_relative": 3.092909 }, "status": "success" }, { "aic": { "tpot_error_pct": 26.85, "tpot_relative": 4.967968, "ttft_error_pct": 44.48, "ttft_relative": 2.779345 }, "aisimulate": { "tpot_error_pct": 27.63, "tpot_relative": 4.998265, "ttft_error_pct": 48.9, "ttft_relative": 2.55804 }, "concurrency": 64, "measured": { "tpot_relative": 3.916258, "ttft_relative": 5.006235 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.59, "tpot_relative": 7.668139, "ttft_error_pct": 65.92, "ttft_relative": 2.805825 }, "aisimulate": { "tpot_error_pct": 24.87, "tpot_relative": 7.810879, "ttft_error_pct": 45.1, "ttft_relative": 4.520385 }, "concurrency": 128, "measured": { "tpot_relative": 6.255322, "ttft_relative": 8.233809 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 3, "tpot_mape_pct": 35.56, "tpot_shape_error_pct": 2.01, "ttft_mape_pct": 79.72, "ttft_shape_error_pct": 19.13 }, "aisimulate": { "points": 3, "status_counts": { "failed": 0, "success": 3, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 34.64, "tpot_shape_error_pct": 3.74, "ttft_mape_pct": 11.96, "ttft_shape_error_pct": 17.17 }, "framework": "sglang", "id": "5933def085ee4c07", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 33.76, "tpot_relative": 1.337621, "ttft_error_pct": 105.99, "ttft_relative": 2.059909 }, "aisimulate": { "tpot_error_pct": 31.36, "tpot_relative": 1.313625, "ttft_error_pct": 0.59, "ttft_relative": 0.994141 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 37.85, "tpot_relative": 1.406848, "ttft_error_pct": 74.17, "ttft_relative": 2.14064 }, "aisimulate": { "tpot_error_pct": 35.13, "tpot_relative": 1.379117, "ttft_error_pct": 15.92, "ttft_relative": 1.033428 }, "concurrency": 2, "measured": { "tpot_relative": 1.020601, "ttft_relative": 1.229045 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.06, "tpot_relative": 1.514243, "ttft_error_pct": 58.99, "ttft_relative": 2.251322 }, "aisimulate": { "tpot_error_pct": 37.42, "tpot_relative": 1.540706, "ttft_error_pct": 19.39, "ttft_relative": 1.141462 }, "concurrency": 4, "measured": { "tpot_relative": 1.121178, "ttft_relative": 1.415991 }, "status": "success" } ], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 11, "tpot_mape_pct": 39.32, "tpot_shape_error_pct": 2.52, "ttft_mape_pct": 52.98, "ttft_shape_error_pct": 62.69 }, "aisimulate": { "points": 11, "status_counts": { "failed": 0, "success": 11, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 31.26, "tpot_shape_error_pct": 8.67, "ttft_mape_pct": 214.61, "ttft_shape_error_pct": 276.18 }, "framework": "sglang", "id": "2e1375a4b92d3143", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 36.45, "tpot_relative": 1.364482, "ttft_error_pct": 71.64, "ttft_relative": 1.716402 }, "aisimulate": { "tpot_error_pct": 33.85, "tpot_relative": 1.338472, "ttft_error_pct": 17.54, "ttft_relative": 0.824603 }, "concurrency": 2, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 37.35, "tpot_relative": 1.627782, "ttft_error_pct": 67.62, "ttft_relative": 1.804993 }, "aisimulate": { "tpot_error_pct": 39.83, "tpot_relative": 1.657192, "ttft_error_pct": 15.37, "ttft_relative": 0.911379 }, "concurrency": 4, "measured": { "tpot_relative": 1.185173, "ttft_relative": 1.076866 }, "status": "success" }, { "aic": { "tpot_error_pct": 40.04, "tpot_relative": 2.043765, "ttft_error_pct": 8.15, "ttft_relative": 1.983338 }, "aisimulate": { "tpot_error_pct": 40.59, "tpot_relative": 2.051891, "ttft_error_pct": 40.88, "ttft_relative": 1.084267 }, "concurrency": 8, "measured": { "tpot_relative": 1.459448, "ttft_relative": 1.833892 }, "status": "success" }, { "aic": { "tpot_error_pct": 40.36, "tpot_relative": 2.752366, "ttft_error_pct": 0.08, "ttft_relative": 2.348776 }, "aisimulate": { "tpot_error_pct": 40.39, "tpot_relative": 2.753058, "ttft_error_pct": 40.97, "ttft_relative": 1.387641 }, "concurrency": 16, "measured": { "tpot_relative": 1.960977, "ttft_relative": 2.350732 }, "status": "success" }, { "aic": { "tpot_error_pct": 35.08, "tpot_relative": 3.684753, "ttft_error_pct": 14.34, "ttft_relative": 3.065286 }, "aisimulate": { "tpot_error_pct": 35.81, "tpot_relative": 3.704706, "ttft_error_pct": 42.65, "ttft_relative": 2.051964 }, "concurrency": 32, "measured": { "tpot_relative": 2.727922, "ttft_relative": 3.578228 }, "status": "success" }, { "aic": { "tpot_error_pct": 37.29, "tpot_relative": 5.440847, "ttft_error_pct": 37.25, "ttft_relative": 3.568125 }, "aisimulate": { "tpot_error_pct": 38.76, "tpot_relative": 5.49916, "ttft_error_pct": 41.29, "ttft_relative": 3.338499 }, "concurrency": 64, "measured": { "tpot_relative": 3.963099, "ttft_relative": 5.686506 }, "status": "success" }, { "aic": { "tpot_error_pct": 36.69, "tpot_relative": 8.394558, "ttft_error_pct": 60.74, "ttft_relative": 3.642048 }, "aisimulate": { "tpot_error_pct": 38.65, "tpot_relative": 8.514698, "ttft_error_pct": 36.4, "ttft_relative": 5.900155 }, "concurrency": 128, "measured": { "tpot_relative": 6.14122, "ttft_relative": 9.277686 }, "status": "success" }, { "aic": { "tpot_error_pct": 39.03, "tpot_relative": 13.997566, "ttft_error_pct": 76.1, "ttft_relative": 3.783231 }, "aisimulate": { "tpot_error_pct": 42.67, "tpot_relative": 14.364224, "ttft_error_pct": 30.32, "ttft_relative": 11.032054 }, "concurrency": 256, "measured": { "tpot_relative": 10.06809, "ttft_relative": 15.831417 }, "status": "success" }, { "aic": { "tpot_error_pct": 41.58, "tpot_relative": 16.747025, "ttft_error_pct": 79.98, "ttft_relative": 3.854876 }, "aisimulate": { "tpot_error_pct": 22.01, "tpot_relative": 14.432372, "ttft_error_pct": 408.7, "ttft_relative": 97.963802 }, "concurrency": 320, "measured": { "tpot_relative": 11.828803, "ttft_relative": 19.257544 }, "status": "success" }, { "aic": { "tpot_error_pct": 43.41, "tpot_relative": 19.439374, "ttft_error_pct": 82.58, "ttft_relative": 3.92561 }, "aisimulate": { "tpot_error_pct": 7.28, "tpot_relative": 14.541358, "ttft_error_pct": 717.04, "ttft_relative": 184.095344 }, "concurrency": 384, "measured": { "tpot_relative": 13.55499, "ttft_relative": 22.532059 }, "status": "success" }, { "aic": { "tpot_error_pct": 45.29, "tpot_relative": 22.053548, "ttft_error_pct": 84.25, "ttft_relative": 3.996179 }, "aisimulate": { "tpot_error_pct": 3.98, "tpot_relative": 14.57388, "ttft_error_pct": 969.57, "ttft_relative": 271.364445 }, "concurrency": 448, "measured": { "tpot_relative": 15.178701, "ttft_relative": 25.371471 }, "status": "success" } ], "precision": "fp8", "rows": 11, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 53.67, "tpot_shape_error_pct": 9.94, "ttft_mape_pct": 38.54, "ttft_shape_error_pct": 12.85 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 56.76, "tpot_shape_error_pct": 9.55, "ttft_mape_pct": 57.62, "ttft_shape_error_pct": 34.05 }, "framework": "trtllm", "id": "f3ae43cbe975cb5d", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 61.7, "tpot_relative": 1.617031, "ttft_error_pct": 42.25, "ttft_relative": 0.577535 }, "aisimulate": { "tpot_error_pct": 64.62, "tpot_relative": 1.646237, "ttft_error_pct": 63.79, "ttft_relative": 0.362105 }, "concurrency": 8, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 45.63, "tpot_relative": 2.62586, "ttft_error_pct": 34.83, "ttft_relative": 0.890561 }, "aisimulate": { "tpot_error_pct": 48.9, "tpot_relative": 2.684804, "ttft_error_pct": 51.46, "ttft_relative": 0.663266 }, "concurrency": 32, "measured": { "tpot_relative": 1.803135, "ttft_relative": 1.366446 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 1, "tpot_mape_pct": 109.2, "tpot_shape_error_pct": null, "ttft_mape_pct": 18.02, "ttft_shape_error_pct": null }, "aisimulate": { "points": 1, "status_counts": { "failed": 0, "success": 1, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 114.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 55.04, "ttft_shape_error_pct": null }, "framework": "trtllm", "id": "49ee369805740573", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 109.2, "tpot_relative": 2.091985, "ttft_error_pct": 18.02, "ttft_relative": 0.819824 }, "aisimulate": { "tpot_error_pct": 114.5, "tpot_relative": 2.145044, "ttft_error_pct": 55.04, "ttft_relative": 0.449596 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" } ], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 2, "tpot_mape_pct": 80.51, "tpot_shape_error_pct": 4.03, "ttft_mape_pct": 24.37, "ttft_shape_error_pct": 0.24 }, "aisimulate": { "points": 2, "status_counts": { "failed": 0, "success": 2, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 84.47, "tpot_shape_error_pct": 4.89, "ttft_mape_pct": 54.55, "ttft_shape_error_pct": 20.32 }, "framework": "trtllm", "id": "f68efba351652086", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2 }, "points": [ { "aic": { "tpot_error_pct": 84.22, "tpot_relative": 1.842184, "ttft_error_pct": 24.47, "ttft_relative": 0.755339 }, "aisimulate": { "tpot_error_pct": 89.1, "tpot_relative": 1.890961, "ttft_error_pct": 58.74, "ttft_relative": 0.412555 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 76.8, "tpot_relative": 2.871507, "ttft_error_pct": 24.28, "ttft_relative": 0.978738 }, "aisimulate": { "tpot_error_pct": 79.85, "tpot_relative": 2.921168, "ttft_error_pct": 50.36, "ttft_relative": 0.641622 }, "concurrency": 16, "measured": { "tpot_relative": 1.624194, "ttft_relative": 1.292626 }, "status": "success" } ], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 29, "tpot_mape_pct": 14.19, "tpot_shape_error_pct": 4.27, "ttft_mape_pct": 30.54, "ttft_shape_error_pct": 38.84 }, "aisimulate": { "coverage_pct": 100.0, "points": 29, "status_counts": { "failed": 0, "success": 29, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 14.97, "tpot_shape_error_pct": 3.94, "ttft_mape_pct": 48.59, "ttft_shape_error_pct": 16.52 }, "gpu": "b300", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 29, "topologies": [ { "aic": { "points": 5, "tpot_mape_pct": 12.07, "tpot_shape_error_pct": 2.74, "ttft_mape_pct": 25.19, "ttft_shape_error_pct": 34.25 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 12.76, "tpot_shape_error_pct": 2.45, "ttft_mape_pct": 52.04, "ttft_shape_error_pct": 14.31 }, "framework": "sglang", "id": "626264e920f685a0", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 9.67, "tpot_relative": 1.09665, "ttft_error_pct": 6.77, "ttft_relative": 1.067658 }, "aisimulate": { "tpot_error_pct": 11.37, "tpot_relative": 1.113731, "ttft_error_pct": 45.84, "ttft_relative": 0.541561 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.15, "tpot_relative": 1.44273, "ttft_error_pct": 12.56, "ttft_relative": 1.173194 }, "aisimulate": { "tpot_error_pct": 10.0, "tpot_relative": 1.44074, "ttft_error_pct": 51.96, "ttft_relative": 0.644549 }, "concurrency": 8, "measured": { "tpot_relative": 1.309727, "ttft_relative": 1.341789 }, "status": "success" }, { "aic": { "tpot_error_pct": 15.53, "tpot_relative": 1.993098, "ttft_error_pct": 25.3, "ttft_relative": 1.390794 }, "aisimulate": { "tpot_error_pct": 15.17, "tpot_relative": 1.986814, "ttft_error_pct": 55.7, "ttft_relative": 0.824766 }, "concurrency": 16, "measured": { "tpot_relative": 1.725157, "ttft_relative": 1.861781 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.18, "tpot_relative": 2.631546, "ttft_error_pct": 32.12, "ttft_relative": 1.815536 }, "aisimulate": { "tpot_error_pct": 10.75, "tpot_relative": 2.645232, "ttft_error_pct": 54.4, "ttft_relative": 1.219722 }, "concurrency": 32, "measured": { "tpot_relative": 2.388437, "ttft_relative": 2.674649 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.8, "tpot_relative": 3.941044, "ttft_error_pct": 49.21, "ttft_relative": 2.113107 }, "aisimulate": { "tpot_error_pct": 16.48, "tpot_relative": 3.998687, "ttft_error_pct": 52.3, "ttft_relative": 1.984499 }, "concurrency": 64, "measured": { "tpot_relative": 3.432823, "ttft_relative": 4.160692 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 5, "tpot_mape_pct": 17.48, "tpot_shape_error_pct": 3.72, "ttft_mape_pct": 26.62, "ttft_shape_error_pct": 37.44 }, "aisimulate": { "points": 5, "status_counts": { "failed": 0, "success": 5, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 18.68, "tpot_shape_error_pct": 3.16, "ttft_mape_pct": 48.65, "ttft_shape_error_pct": 19.23 }, "framework": "sglang", "id": "cd6c91133f8e4705", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 15.2, "tpot_relative": 1.152009, "ttft_error_pct": 19.67, "ttft_relative": 1.196696 }, "aisimulate": { "tpot_error_pct": 17.31, "tpot_relative": 1.173116, "ttft_error_pct": 39.32, "ttft_relative": 0.606837 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.41, "tpot_relative": 1.543414, "ttft_error_pct": 6.44, "ttft_relative": 1.314994 }, "aisimulate": { "tpot_error_pct": 18.78, "tpot_relative": 1.548241, "ttft_error_pct": 41.55, "ttft_relative": 0.722138 }, "concurrency": 8, "measured": { "tpot_relative": 1.303479, "ttft_relative": 1.235487 }, "status": "success" }, { "aic": { "tpot_error_pct": 22.16, "tpot_relative": 2.210208, "ttft_error_pct": 24.31, "ttft_relative": 1.556575 }, "aisimulate": { "tpot_error_pct": 22.32, "tpot_relative": 2.212967, "ttft_error_pct": 55.07, "ttft_relative": 0.924096 }, "concurrency": 16, "measured": { "tpot_relative": 1.809208, "ttft_relative": 2.056577 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.33, "tpot_relative": 3.019957, "ttft_error_pct": 34.07, "ttft_relative": 2.031909 }, "aisimulate": { "tpot_error_pct": 13.33, "tpot_relative": 3.046757, "ttft_error_pct": 55.65, "ttft_relative": 1.36658 }, "concurrency": 32, "measured": { "tpot_relative": 2.688426, "ttft_relative": 3.081696 }, "status": "success" }, { "aic": { "tpot_error_pct": 19.28, "tpot_relative": 4.67839, "ttft_error_pct": 48.6, "ttft_relative": 2.365153 }, "aisimulate": { "tpot_error_pct": 21.67, "tpot_relative": 4.77202, "ttft_error_pct": 51.68, "ttft_relative": 2.223425 }, "concurrency": 64, "measured": { "tpot_relative": 3.922141, "ttft_relative": 4.601094 }, "status": "success" } ], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 7.78, "tpot_shape_error_pct": 3.56, "ttft_mape_pct": 42.11, "ttft_shape_error_pct": 43.14 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 7.58, "tpot_shape_error_pct": 3.31, "ttft_mape_pct": 60.71, "ttft_shape_error_pct": 15.9 }, "framework": "sglang", "id": "e98818f32401758e", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 8.13, "tpot_relative": 0.918744, "ttft_error_pct": 9.62, "ttft_relative": 0.903822 }, "aisimulate": { "tpot_error_pct": 6.72, "tpot_relative": 0.932764, "ttft_error_pct": 54.71, "ttft_relative": 0.452888 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.72, "tpot_relative": 1.198292, "ttft_error_pct": 20.4, "ttft_relative": 0.993471 }, "aisimulate": { "tpot_error_pct": 4.26, "tpot_relative": 1.191673, "ttft_error_pct": 56.83, "ttft_relative": 0.538712 }, "concurrency": 8, "measured": { "tpot_relative": 1.244636, "ttft_relative": 1.248019 }, "status": "success" }, { "aic": { "tpot_error_pct": 3.32, "tpot_relative": 1.621969, "ttft_error_pct": 39.38, "ttft_relative": 1.177866 }, "aisimulate": { "tpot_error_pct": 4.04, "tpot_relative": 1.609971, "ttft_error_pct": 64.51, "ttft_relative": 0.689483 }, "concurrency": 16, "measured": { "tpot_relative": 1.677671, "ttft_relative": 1.942939 }, "status": "success" }, { "aic": { "tpot_error_pct": 10.01, "tpot_relative": 2.142591, "ttft_error_pct": 48.03, "ttft_relative": 1.538259 }, "aisimulate": { "tpot_error_pct": 10.08, "tpot_relative": 2.141086, "ttft_error_pct": 65.56, "ttft_relative": 1.019534 }, "concurrency": 32, "measured": { "tpot_relative": 2.38105, "ttft_relative": 2.959983 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.61, "tpot_relative": 3.104729, "ttft_error_pct": 61.25, "ttft_relative": 1.79126 }, "aisimulate": { "tpot_error_pct": 11.18, "tpot_relative": 3.119769, "ttft_error_pct": 64.12, "ttft_relative": 1.658748 }, "concurrency": 64, "measured": { "tpot_relative": 3.512627, "ttft_relative": 4.622786 }, "status": "success" }, { "aic": { "tpot_error_pct": 9.89, "tpot_relative": 4.940875, "ttft_error_pct": 73.99, "ttft_relative": 1.838184 }, "aisimulate": { "tpot_error_pct": 9.18, "tpot_relative": 4.979713, "ttft_error_pct": 58.53, "ttft_relative": 2.931373 }, "concurrency": 128, "measured": { "tpot_relative": 5.483056, "ttft_relative": 7.068248 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 6, "tpot_mape_pct": 18.34, "tpot_shape_error_pct": 7.36, "ttft_mape_pct": 30.56, "ttft_shape_error_pct": 38.17 }, "aisimulate": { "points": 6, "status_counts": { "failed": 0, "success": 6, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 19.04, "tpot_shape_error_pct": 5.91, "ttft_mape_pct": 50.93, "ttft_shape_error_pct": 9.02 }, "framework": "sglang", "id": "ae945a8d2f58b720", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 11.5, "tpot_relative": 1.115015, "ttft_error_pct": 3.6, "ttft_relative": 1.035985 }, "aisimulate": { "tpot_error_pct": 13.45, "tpot_relative": 1.134538, "ttft_error_pct": 47.55, "ttft_relative": 0.524489 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 16.52, "tpot_relative": 1.456076, "ttft_error_pct": 7.95, "ttft_relative": 1.138662 }, "aisimulate": { "tpot_error_pct": 16.19, "tpot_relative": 1.451922, "ttft_error_pct": 49.54, "ttft_relative": 0.624205 }, "concurrency": 8, "measured": { "tpot_relative": 1.249605, "ttft_relative": 1.236953 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.77, "tpot_relative": 1.99243, "ttft_error_pct": 22.58, "ttft_relative": 1.349504 }, "aisimulate": { "tpot_error_pct": 21.22, "tpot_relative": 1.983321, "ttft_error_pct": 54.18, "ttft_relative": 0.798739 }, "concurrency": 16, "measured": { "tpot_relative": 1.636163, "ttft_relative": 1.743023 }, "status": "success" }, { "aic": { "tpot_error_pct": 20.02, "tpot_relative": 2.645714, "ttft_error_pct": 31.38, "ttft_relative": 1.762219 }, "aisimulate": { "tpot_error_pct": 20.33, "tpot_relative": 2.652492, "ttft_error_pct": 54.0, "ttft_relative": 1.181231 }, "concurrency": 32, "measured": { "tpot_relative": 2.204362, "ttft_relative": 2.56817 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.83, "tpot_relative": 3.903637, "ttft_error_pct": 51.36, "ttft_relative": 2.051852 }, "aisimulate": { "tpot_error_pct": 19.93, "tpot_relative": 3.939833, "ttft_error_pct": 54.44, "ttft_relative": 1.921876 }, "concurrency": 64, "measured": { "tpot_relative": 3.284993, "ttft_relative": 4.2186 }, "status": "success" }, { "aic": { "tpot_error_pct": 21.41, "tpot_relative": 6.224685, "ttft_error_pct": 66.47, "ttft_relative": 2.102955 }, "aisimulate": { "tpot_error_pct": 23.13, "tpot_relative": 6.312812, "ttft_error_pct": 45.85, "ttft_relative": 3.396904 }, "concurrency": 128, "measured": { "tpot_relative": 5.126955, "ttft_relative": 6.272735 }, "status": "success" } ], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none" }, { "aic": { "points": 7, "tpot_mape_pct": 15.3, "tpot_shape_error_pct": 3.67, "ttft_mape_pct": 27.22, "ttft_shape_error_pct": 39.82 }, "aisimulate": { "points": 7, "status_counts": { "failed": 0, "success": 7, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 16.74, "tpot_shape_error_pct": 4.35, "ttft_mape_pct": 33.68, "ttft_shape_error_pct": 22.96 }, "framework": "sglang", "id": "327c76f9802ef801", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4 }, "points": [ { "aic": { "tpot_error_pct": 19.05, "tpot_relative": 1.190493, "ttft_error_pct": 18.54, "ttft_relative": 1.185437 }, "aisimulate": { "tpot_error_pct": 21.26, "tpot_relative": 1.212604, "ttft_error_pct": 40.02, "ttft_relative": 0.599782 }, "concurrency": 4, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 17.36, "tpot_relative": 1.472418, "ttft_error_pct": 8.1, "ttft_relative": 1.30264 }, "aisimulate": { "tpot_error_pct": 17.81, "tpot_relative": 1.47803, "ttft_error_pct": 49.65, "ttft_relative": 0.713649 }, "concurrency": 8, "measured": { "tpot_relative": 1.254606, "ttft_relative": 1.4174 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.88, "tpot_relative": 2.007986, "ttft_error_pct": 14.74, "ttft_relative": 1.541948 }, "aisimulate": { "tpot_error_pct": 18.97, "tpot_relative": 2.009481, "ttft_error_pct": 49.5, "ttft_relative": 0.913281 }, "concurrency": 16, "measured": { "tpot_relative": 1.689124, "ttft_relative": 1.808455 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.27, "tpot_relative": 2.696424, "ttft_error_pct": 4.39, "ttft_relative": 2.012971 }, "aisimulate": { "tpot_error_pct": 11.97, "tpot_relative": 2.713382, "ttft_error_pct": 35.85, "ttft_relative": 1.350545 }, "concurrency": 32, "measured": { "tpot_relative": 2.423418, "ttft_relative": 2.105292 }, "status": "success" }, { "aic": { "tpot_error_pct": 11.67, "tpot_relative": 3.972598, "ttft_error_pct": 25.36, "ttft_relative": 2.343504 }, "aisimulate": { "tpot_error_pct": 13.05, "tpot_relative": 4.021647, "ttft_error_pct": 30.01, "ttft_relative": 2.197321 }, "concurrency": 64, "measured": { "tpot_relative": 3.557293, "ttft_relative": 3.139586 }, "status": "success" }, { "aic": { "tpot_error_pct": 13.95, "tpot_relative": 6.127405, "ttft_error_pct": 49.08, "ttft_relative": 2.392404 }, "aisimulate": { "tpot_error_pct": 15.92, "tpot_relative": 6.233487, "ttft_error_pct": 17.35, "ttft_relative": 3.883485 }, "concurrency": 128, "measured": { "tpot_relative": 5.377204, "ttft_relative": 4.698568 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.92, "tpot_relative": 10.072513, "ttft_error_pct": 70.34, "ttft_relative": 2.48617 }, "aisimulate": { "tpot_error_pct": 18.22, "tpot_relative": 10.361473, "ttft_error_pct": 13.37, "ttft_relative": 7.261558 }, "concurrency": 256, "measured": { "tpot_relative": 8.764593, "ttft_relative": 8.381899 }, "status": "success" } ], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none" } ] }, { "aic": { "points": 4, "tpot_mape_pct": 16.81, "tpot_shape_error_pct": 8.37, "ttft_mape_pct": 27.38, "ttft_shape_error_pct": 26.36 }, "aisimulate": { "coverage_pct": 100.0, "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.33, "tpot_shape_error_pct": 11.06, "ttft_mape_pct": 63.02, "ttft_shape_error_pct": 25.36 }, "gpu": "h100", "precisions": [ "fp8" ], "rows": 4, "topologies": [ { "aic": { "points": 4, "tpot_mape_pct": 16.81, "tpot_shape_error_pct": 8.37, "ttft_mape_pct": 27.38, "ttft_shape_error_pct": 26.36 }, "aisimulate": { "points": 4, "status_counts": { "failed": 0, "success": 4, "unknown": 0, "unsupported": 0 }, "tpot_mape_pct": 15.33, "tpot_shape_error_pct": 11.06, "ttft_mape_pct": 63.02, "ttft_shape_error_pct": 25.36 }, "framework": "sglang", "id": "bff0a3349e5f36a9", "parallelism": { "attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8 }, "points": [ { "aic": { "tpot_error_pct": 21.72, "tpot_relative": 0.782799, "ttft_error_pct": 35.39, "ttft_relative": 0.646124 }, "aisimulate": { "tpot_error_pct": 21.81, "tpot_relative": 0.781878, "ttft_error_pct": 68.64, "ttft_relative": 0.313572 }, "concurrency": 1, "measured": { "tpot_relative": 1.0, "ttft_relative": 1.0 }, "status": "success" }, { "aic": { "tpot_error_pct": 14.72, "tpot_relative": 0.874292, "ttft_error_pct": 19.21, "ttft_relative": 0.665566 }, "aisimulate": { "tpot_error_pct": 15.04, "tpot_relative": 0.870992, "ttft_error_pct": 60.41, "ttft_relative": 0.326169 }, "concurrency": 2, "measured": { "tpot_relative": 1.025224, "ttft_relative": 0.823841 }, "status": "success" }, { "aic": { "tpot_error_pct": 18.14, "tpot_relative": 1.031574, "ttft_error_pct": 10.0, "ttft_relative": 0.700003 }, "aisimulate": { "tpot_error_pct": 14.35, "tpot_relative": 1.079326, "ttft_error_pct": 53.71, "ttft_relative": 0.360072 }, "concurrency": 4, "measured": { "tpot_relative": 1.260199, "ttft_relative": 0.777815 }, "status": "success" }, { "aic": { "tpot_error_pct": 12.65, "tpot_relative": 1.416496, "ttft_error_pct": 44.93, "ttft_relative": 0.769576 }, "aisimulate": { "tpot_error_pct": 10.1, "tpot_relative": 1.457831, "ttft_error_pct": 69.33, "ttft_relative": 0.428639 }, "concurrency": 8, "measured": { "tpot_relative": 1.621622, "ttft_relative": 1.397434 }, "status": "success" } ], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none" } ] } ], "identity": "8192:1024", "label": "8k1k", "precisions": [ "bf16", "fp4", "fp8" ], "rows": 64 } ] } ], "schema_version": 1, "scope": { "claim": "Accuracy applies only to the exact measured operating points in this snapshot; it is not universal model, hardware, or deployment support.", "excluded_multinode_rows": 385, "latency_scope": "client_observed", "measurement_scope": "end_to_end", "metrics": [ "TTFT", "TPOT" ], "multinode": "excluded", "published_rows": 807, "raw_rows": 1192 }, "snapshot": { "aic_commit_sha": "e46be717175acf06bdbbdeadb7aaf9bb2afdae8d", "aic_source": { "branch": "main", "commit_sha": "e46be717175acf06bdbbdeadb7aaf9bb2afdae8d", "repository": "https://github.com/ai-dynamo/aisimulate" }, "aisimulate_completed_at": "2026-09-15T05:11:52.184879+00:00", "aisimulate_method": "randomized_synthetic_engine_replay", "aisimulate_packages": { "aisimulate": "0.12.0" }, "aisimulate_sot_sha256": "47fdffa637a89c230aa6d3bcb76918fca1b09cb90019337f1e0b67d12bedf51b", "corrections": [], "evaluated_revision": { "branch": "main", "commit_sha": "e46be717175acf06bdbbdeadb7aaf9bb2afdae8d" }, "measurement_date_through": "2026-09-13", "measurement_source": "SemiAnalysis InferenceX", "measurement_source_url": "https://github.com/SemiAnalysisAI/InferenceX-app/releases/tag/db-dump/2026-09-14", "prediction_generated_at": "2026-09-15T05:06:47.245779+00:00", "predictions_sha256": "08c17d69203ad3abafbefd903c046400c58f3e26af15cdb16988f186f60010bc", "release_tag": "db-dump/2026-09-14" }, "title": "AISimulate E2E Accuracy Overview", "totals": { "aic": { "points": 807, "tpot_mape_pct": 32.11, "tpot_shape_error_pct": 17.33, "ttft_mape_pct": 49.01, "ttft_shape_error_pct": 36.52 }, "aisimulate": { "coverage_pct": 95.54, "points": 771, "status_counts": { "failed": 18, "success": 771, "unknown": 0, "unsupported": 18 }, "tpot_mape_pct": 28.91, "tpot_shape_error_pct": 16.0, "ttft_mape_pct": 48.49, "ttft_shape_error_pct": 26.27 }, "frameworks": [ "dynamo-sglang", "dynamo-vllm", "sglang", "trtllm", "vllm" ], "gpu_skus": [ "b200", "b300", "h100", "h200" ], "models": 8, "precisions": [ "bf16", "fp4", "fp8" ], "rows": 807, "workloads": [ "1024:1024", "1024:8192", "8192:1024" ] } }