{"models": [{"aic": {"coverage_pct": 81.36, "points": 96, "status_counts": {"failed": 22, "success": 96, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.92, "tpot_shape_error_pct": 6.08, "ttft_mape_pct": 39.42, "ttft_shape_error_pct": 30.47}, "aisimulate": {"coverage_pct": 59.32, "points": 70, "status_counts": {"failed": 48, "success": 70, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.92, "tpot_shape_error_pct": 5.94, "ttft_mape_pct": 57.66, "ttft_shape_error_pct": 49.98}, "frameworks": ["dynamo-trt", "sglang", "trt"], "gpu_skus": ["b200", "b300", "gb300", "h200"], "hf_model_paths": ["deepseek-ai/DeepSeek-R1-0528"], "model": "deepseek-ai/DeepSeek-R1-0528", "precisions": ["fp8"], "rows": 118, "workloads": [{"aic": {"coverage_pct": 79.63, "points": 43, "status_counts": {"failed": 11, "success": 43, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.76, "tpot_shape_error_pct": 5.71, "ttft_mape_pct": 47.54, "ttft_shape_error_pct": 30.51}, "aisimulate": {"coverage_pct": 59.26, "points": 32, "status_counts": {"failed": 22, "success": 32, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.82, "tpot_shape_error_pct": 5.29, "ttft_mape_pct": 57.13, "ttft_shape_error_pct": 30.81}, "gpu_skus": ["b200", "b300", "gb300", "h200"], "gpus": [{"aic": {"coverage_pct": 80.95, "points": 17, "status_counts": {"failed": 4, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.02, "tpot_shape_error_pct": 3.46, "ttft_mape_pct": 47.28, "ttft_shape_error_pct": 27.37}, "aisimulate": {"coverage_pct": 71.43, "points": 15, "status_counts": {"failed": 6, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.11, "tpot_shape_error_pct": 3.39, "ttft_mape_pct": 64.68, "ttft_shape_error_pct": 26.58}, "gpu": "b200", "precisions": ["fp8"], "rows": 21, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "cbc08806dc16cacf", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5152, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 38267.735591296594, "interactivity_tok_s": 26.632535202206086, "output_per_gpu": 2960.6827453904752, "total_per_gpu": 4231.307930607052, "tpot_ms": 37.548058883900985, "tpot_relative": 1.0, "ttft_ms": 3713.01361483936, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "7eab062cefa3efde", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 8595.887054229388, "interactivity_tok_s": 142.27793190470896, "output_per_gpu": 17.188984928322284, "total_per_gpu": 25.909553673285156, "tpot_ms": 7.028496876590481, "tpot_relative": 1.0, "ttft_ms": 2153.445836302126, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 10473.337895181612, "interactivity_tok_s": 90.79868676602139, "output_per_gpu": 114.29174541937913, "total_per_gpu": 171.16646711978413, "tpot_ms": 11.013375144696687, "tpot_relative": 1.56696, "ttft_ms": 312.62901681184303, "ttft_relative": 0.145176, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 16897.80101022434, "interactivity_tok_s": 56.11247618445586, "output_per_gpu": 281.55247356061943, "total_per_gpu": 422.79260446706013, "tpot_ms": 17.82134861973918, "tpot_relative": 2.535585, "ttft_ms": 524.2403284548345, "ttft_relative": 0.243443, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.83, "tpot_shape_error_pct": null, "ttft_mape_pct": 92.65, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "b2bc5176f08927ec", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 30989.323, "interactivity_tok_s": 33.19942896982172, "output_per_gpu": 1222.9376041666667, "total_per_gpu": null, "tpot_error_pct": 5.83, "tpot_ms": 30.121, "tpot_relative": 0.941703, "ttft_error_pct": 92.65, "ttft_ms": 175.54, "ttft_relative": 0.073538, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1920, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 31860.05924793203, "interactivity_tok_s": 31.263986049421515, "output_per_gpu": 1327.342381141612, "total_per_gpu": 2211.5519930903147, "tpot_ms": 31.985684692259618, "tpot_relative": 1.0, "ttft_ms": 2387.065498391397, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.79, "tpot_shape_error_pct": null, "ttft_mape_pct": 95.89, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "bca091b51014a5d2", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 73253.545, "interactivity_tok_s": 13.998740113389795, "output_per_gpu": 3300.224, "total_per_gpu": null, "tpot_error_pct": 3.79, "tpot_ms": 71.435, "tpot_relative": 0.962061, "ttft_error_pct": 95.89, "ttft_ms": 175.54, "ttft_relative": 0.041127, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 72608.23831600661, "interactivity_tok_s": 13.467641436349139, "output_per_gpu": 6256.290108712145, "total_per_gpu": 6258.563142211024, "tpot_ms": 74.25205109047542, "tpot_relative": 1.0, "ttft_ms": 4268.25473396192, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.87, "tpot_shape_error_pct": 3.13, "ttft_mape_pct": 31.08, "ttft_shape_error_pct": 32.27}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.85, "tpot_shape_error_pct": 3.14, "ttft_mape_pct": 66.82, "ttft_shape_error_pct": 36.98}, "framework": "sglang", "id": "5360c5744950bac9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7468.068550570578, "interactivity_tok_s": 139.84404216908717, "output_per_gpu": 17.29045092160621, "total_per_gpu": null, "tpot_error_pct": 0.45, "tpot_ms": 7.150823048942532, "tpot_relative": 0.995458, "ttft_error_pct": 4.73, "ttft_ms": 152.77657150236755, "ttft_relative": 0.952715, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6642.073657600001, "interactivity_tok_s": 140.2211575288165, "output_per_gpu": 17.340217217888622, "total_per_gpu": 34.55810818017026, "tpot_error_pct": 0.72, "tpot_ms": 7.131591391937358, "tpot_relative": 0.992781, "ttft_error_pct": 51.42, "ttft_ms": 77.90486280000023, "ttft_relative": 0.485815, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6772.038815054111, "interactivity_tok_s": 139.20885924852723, "output_per_gpu": 17.00607487945968, "total_per_gpu": 33.89218070452768, "tpot_ms": 7.18345086223799, "tpot_relative": 1.0, "ttft_ms": 160.35921096336097, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7772.112384458593, "interactivity_tok_s": 134.34067038969985, "output_per_gpu": 33.23346960820047, "total_per_gpu": null, "tpot_error_pct": 6.11, "tpot_ms": 7.443762168963182, "tpot_relative": 1.036238, "ttft_error_pct": 11.67, "ttft_ms": 157.1436856092576, "ttft_relative": 0.979948, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7034.581699349999, "interactivity_tok_s": 134.33859644670937, "output_per_gpu": 33.134332771477304, "total_per_gpu": 65.66284133510402, "tpot_error_pct": 6.1, "tpot_ms": 7.443877087078908, "tpot_relative": 1.036254, "ttft_error_pct": 55.34, "ttft_ms": 79.46094910000043, "ttft_relative": 0.495518, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7585.5596155743115, "interactivity_tok_s": 126.13817246773432, "output_per_gpu": 30.741316768390774, "total_per_gpu": 60.920562949516665, "tpot_ms": 7.927814240814344, "tpot_relative": 1.103622, "ttft_ms": 177.90619117440656, "ttft_relative": 1.109423, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8816.84009680788, "interactivity_tok_s": 118.24383140708963, "output_per_gpu": 57.62410225329217, "total_per_gpu": null, "tpot_error_pct": 4.09, "tpot_ms": 8.457100789953278, "tpot_relative": 1.177303, "ttft_error_pct": 18.96, "ttft_ms": 165.22598868567704, "ttft_relative": 1.030349, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7951.111498725004, "interactivity_tok_s": 116.51518557673428, "output_per_gpu": 56.03712706844695, "total_per_gpu": 112.62227583777305, "tpot_error_pct": 2.67, "tpot_ms": 8.58257226343619, "tpot_relative": 1.19477, "ttft_error_pct": 59.57, "ttft_ms": 82.42816115000021, "ttft_relative": 0.514022, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8288.363852270413, "interactivity_tok_s": 113.40642946071962, "output_per_gpu": 53.719025339394335, "total_per_gpu": 107.96340223009334, "tpot_ms": 8.817842204849313, "tpot_relative": 1.227522, "ttft_ms": 203.87336127460003, "ttft_relative": 1.271354, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10727.934680665243, "interactivity_tok_s": 96.99866261636475, "output_per_gpu": 95.0056409473427, "total_per_gpu": null, "tpot_error_pct": 5.92, "tpot_ms": 10.309420491239731, "tpot_relative": 1.435163, "ttft_error_pct": 27.07, "ttft_ms": 181.39751812699723, "ttft_relative": 1.131195, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9792.77129177501, "interactivity_tok_s": 95.67328347059842, "output_per_gpu": 93.10606659969991, "total_per_gpu": 185.52441651125255, "tpot_error_pct": 4.62, "tpot_ms": 10.4522387413129, "tpot_relative": 1.455044, "ttft_error_pct": 64.8, "ttft_ms": 87.55040664999964, "ttft_relative": 0.545964, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 10424.356947359047, "interactivity_tok_s": 91.25637351884296, "output_per_gpu": 87.30558336349652, "total_per_gpu": 173.96629460597805, "tpot_ms": 10.958138718864564, "tpot_relative": 1.52547, "ttft_ms": 248.7439473217819, "ttft_relative": 1.551167, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13712.169162572747, "interactivity_tok_s": 75.78675490602012, "output_per_gpu": 149.19046810465758, "total_per_gpu": null, "tpot_error_pct": 0.83, "tpot_ms": 13.19491778266607, "tpot_relative": 1.836849, "ttft_error_pct": 41.99, "ttft_ms": 213.7682709053568, "ttft_relative": 1.333059, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12363.242823562494, "interactivity_tok_s": 74.62035633325965, "output_per_gpu": 145.38383306354388, "total_per_gpu": 292.31532640140404, "tpot_error_pct": 2.4, "tpot_ms": 13.401168918758993, "tpot_relative": 1.865561, "ttft_error_pct": 73.73, "ttft_ms": 96.80373541249985, "ttft_relative": 0.603668, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 12347.64785610605, "interactivity_tok_s": 76.41284043846744, "output_per_gpu": 145.25332550333488, "total_per_gpu": 292.0529219836881, "tpot_ms": 13.086805754921054, "tpot_relative": 1.821799, "ttft_ms": 368.4929422743153, "ttft_relative": 2.297922, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16471.995187807857, "interactivity_tok_s": 63.17398463169077, "output_per_gpu": 249.45143279174454, "total_per_gpu": null, "tpot_error_pct": 2.36, "tpot_ms": 15.829300713419892, "tpot_relative": 2.203579, "ttft_error_pct": 40.84, "ttft_ms": 278.6205579793071, "ttft_relative": 1.737478, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15039.478408140343, "interactivity_tok_s": 61.85675022944635, "output_per_gpu": 240.38273151846923, "total_per_gpu": 480.00506959497244, "tpot_error_pct": 0.28, "tpot_ms": 16.16638436857226, "tpot_relative": 2.250504, "ttft_error_pct": 75.68, "ttft_ms": 114.51465118750056, "ttft_relative": 0.714113, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 15437.947879653075, "interactivity_tok_s": 61.684004366071, "output_per_gpu": 234.00584429846677, "total_per_gpu": 467.2714669168558, "tpot_ms": 16.211658277977254, "tpot_relative": 2.256806, "ttft_ms": 470.9585539327236, "ttft_relative": 2.936897, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22625.65659717818, "interactivity_tok_s": 45.87057291939417, "output_per_gpu": 363.7153667582764, "total_per_gpu": null, "tpot_error_pct": 0.32, "tpot_ms": 21.800468935874093, "tpot_relative": 3.034818, "ttft_error_pct": 72.33, "ttft_ms": 323.77687577898223, "ttft_relative": 2.019073, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20814.494175456377, "interactivity_tok_s": 44.60278106173327, "output_per_gpu": 346.53658484704056, "total_per_gpu": 693.2294151604243, "tpot_error_pct": 3.17, "tpot_ms": 22.42012664223633, "tpot_relative": 3.12108, "ttft_error_pct": 87.18, "ttft_ms": 149.98362120937495, "ttft_relative": 0.935298, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 21191.840788035915, "interactivity_tok_s": 46.01805795246199, "output_per_gpu": 338.96887775153016, "total_per_gpu": 678.0905888623533, "tpot_ms": 21.730599779613247, "tpot_relative": 3.025092, "ttft_ms": 1170.3175603746786, "ttft_relative": 7.2981, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.88, "tpot_shape_error_pct": 2.56, "ttft_mape_pct": 52.54, "ttft_shape_error_pct": 27.71}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.63, "tpot_shape_error_pct": 2.78, "ttft_mape_pct": 63.94, "ttft_shape_error_pct": 19.97}, "framework": "trt", "id": "226991f51ee839a0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 13096.736203477123, "interactivity_tok_s": 78.73470658298491, "output_per_gpu": 38.98804097997696, "total_per_gpu": null, "tpot_error_pct": 46.16, "tpot_ms": 12.70087923609671, "tpot_relative": 1.461561, "ttft_error_pct": 45.38, "ttft_ms": 103.7367449501888, "ttft_relative": 0.546247, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11766.779614900035, "interactivity_tok_s": 78.34527431810281, "output_per_gpu": 37.86495038596694, "total_per_gpu": 76.10020552522495, "tpot_error_pct": 46.88, "tpot_ms": 12.764011725068853, "tpot_relative": 1.468826, "ttft_error_pct": 65.74, "ttft_ms": 65.06608142500127, "ttft_relative": 0.342618, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 8158.019682427403, "interactivity_tok_s": 115.07556247986619, "output_per_gpu": 54.445916082623164, "total_per_gpu": 109.4242924304037, "tpot_ms": 8.689942316597076, "tpot_relative": 1.0, "ttft_ms": 189.9082841584459, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16042.917568006074, "interactivity_tok_s": 64.22294733680738, "output_per_gpu": 63.73916519558173, "total_per_gpu": null, "tpot_error_pct": 43.34, "tpot_ms": 15.570758451113333, "tpot_relative": 1.791814, "ttft_error_pct": 36.99, "ttft_ms": 114.03167251713471, "ttft_relative": 0.600457, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14621.97704940006, "interactivity_tok_s": 63.816000283216184, "output_per_gpu": 62.32129601634014, "total_per_gpu": 124.18226332518935, "tpot_error_pct": 44.25, "tpot_ms": 15.670051328224707, "tpot_relative": 1.80324, "ttft_error_pct": 58.81, "ttft_ms": 74.54911597500067, "ttft_relative": 0.392553, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 10267.372332402738, "interactivity_tok_s": 92.0564496896869, "output_per_gpu": 88.58934866829969, "total_per_gpu": 176.524342838592, "tpot_ms": 10.862899920330408, "tpot_relative": 1.250054, "ttft_ms": 180.97360140527599, "ttft_relative": 0.952953, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30422.884349774897, "interactivity_tok_s": 33.85643362857571, "output_per_gpu": 269.99167260750755, "total_per_gpu": null, "tpot_error_pct": 47.21, "tpot_ms": 29.536483699688144, "tpot_relative": 3.398927, "ttft_error_pct": 63.61, "ttft_ms": 207.06152499392428, "ttft_relative": 1.090324, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28501.13275771081, "interactivity_tok_s": 32.50065029245585, "output_per_gpu": 252.4625709814985, "total_per_gpu": 505.0389715956016, "tpot_error_pct": 53.35, "tpot_ms": 30.76861512005263, "tpot_relative": 3.540716, "ttft_error_pct": 72.4, "ttft_ms": 157.03709644843718, "ttft_relative": 0.82691, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 19049.675204747473, "interactivity_tok_s": 49.84058876384974, "output_per_gpu": 374.6352403935834, "total_per_gpu": 749.4393952983713, "tpot_ms": 20.063968440222716, "tpot_relative": 2.308872, "ttft_ms": 569.0219993219216, "ttft_relative": 2.996299, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35055.16150768362, "interactivity_tok_s": 29.360156143039408, "output_per_gpu": 468.5401893230547, "total_per_gpu": null, "tpot_error_pct": 38.81, "tpot_ms": 34.059764366650896, "tpot_relative": 3.919447, "ttft_error_pct": 64.16, "ttft_ms": 212.02256059974493, "ttft_relative": 1.116447, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34092.295849771865, "interactivity_tok_s": 27.165070511284426, "output_per_gpu": 420.5265347834942, "total_per_gpu": 841.9768987293999, "tpot_error_pct": 50.02, "tpot_ms": 36.81197880876466, "tpot_relative": 4.236159, "ttft_error_pct": 58.81, "ttft_ms": 243.7087942937488, "ttft_relative": 1.283297, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 23148.990463491828, "interactivity_tok_s": 40.753732173581874, "output_per_gpu": 613.7568655816413, "total_per_gpu": 1228.8620562846602, "tpot_ms": 24.537629970691082, "tpot_relative": 2.823682, "ttft_ms": 591.6610352298449, "ttft_relative": 3.115509, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.23, "tpot_shape_error_pct": 6.95, "ttft_mape_pct": 66.44, "ttft_shape_error_pct": 18.86}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.83, "tpot_shape_error_pct": 5.78, "ttft_mape_pct": 69.7, "ttft_shape_error_pct": 6.06}, "framework": "trt", "id": "ef6ffe29b26f2a38", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 14918.59771666392, "interactivity_tok_s": 69.02678076489424, "output_per_gpu": 137.29146479905643, "total_per_gpu": null, "tpot_error_pct": 21.5, "tpot_ms": 14.487130776183927, "tpot_relative": 0.785024, "ttft_error_pct": 62.95, "ttft_ms": 98.26293262776403, "ttft_relative": 0.370487, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13503.139325206259, "interactivity_tok_s": 68.18708996681227, "output_per_gpu": 132.86569646152378, "total_per_gpu": 267.1457933821618, "tpot_error_pct": 20.53, "tpot_ms": 14.66553273481411, "tpot_relative": 0.794691, "ttft_error_pct": 68.75, "ttft_ms": 82.8858001937505, "ttft_relative": 0.312509, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 17153.07584721013, "interactivity_tok_s": 54.1876548880918, "output_per_gpu": 104.46613964730012, "total_per_gpu": 210.04435682713194, "tpot_ms": 18.4543878502437, "tpot_relative": 1.0, "ttft_ms": 265.22656043380266, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17357.32199674982, "interactivity_tok_s": 59.37816948798006, "output_per_gpu": 236.4382585755195, "total_per_gpu": null, "tpot_error_pct": 26.96, "tpot_ms": 16.841206265249223, "tpot_relative": 0.912585, "ttft_error_pct": 69.94, "ttft_ms": 128.7679873998623, "ttft_relative": 0.485502, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16057.430114406252, "interactivity_tok_s": 57.92532844024433, "output_per_gpu": 224.69618034203765, "total_per_gpu": 448.68158790564974, "tpot_error_pct": 25.12, "tpot_ms": 17.26360517802844, "tpot_relative": 0.935474, "ttft_error_pct": 70.64, "ttft_ms": 125.74789425625026, "ttft_relative": 0.474115, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 21713.297355336545, "interactivity_tok_s": 43.37213284126851, "output_per_gpu": 166.18195293023743, "total_per_gpu": 331.8382288853318, "tpot_ms": 23.05627910113984, "tpot_relative": 1.249366, "ttft_ms": 428.35063492821064, "ttft_relative": 1.615037, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.83, "tpot_shape_error_pct": 4.64, "ttft_mape_pct": 27.33, "ttft_shape_error_pct": 5.38}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.77, "tpot_shape_error_pct": 4.32, "ttft_mape_pct": 53.65, "ttft_shape_error_pct": 4.51}, "framework": "trt", "id": "db390b7b96ba8a08", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 18576.091456233105, "interactivity_tok_s": 55.48187950890463, "output_per_gpu": 110.08331233835055, "total_per_gpu": null, "tpot_error_pct": 38.03, "tpot_ms": 18.02390273818146, "tpot_relative": 1.380325, "ttft_error_pct": 29.23, "ttft_ms": 137.6389550734749, "ttft_relative": 0.707686, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16910.448056774978, "interactivity_tok_s": 55.188560273208466, "output_per_gpu": 107.89589229099907, "total_per_gpu": 214.99482463705615, "tpot_error_pct": 38.77, "tpot_ms": 18.11969718089302, "tpot_relative": 1.387661, "ttft_error_pct": 54.67, "ttft_ms": 88.16697506249986, "ttft_relative": 0.45332, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 12317.768128117314, "interactivity_tok_s": 76.58302096133387, "output_per_gpu": 147.77237122917396, "total_per_gpu": 294.4532397297628, "tpot_ms": 13.057724642449031, "tpot_relative": 1.0, "ttft_ms": 194.49161399388686, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22950.196481929466, "interactivity_tok_s": 44.892814034600626, "output_per_gpu": 178.47859227179717, "total_per_gpu": null, "tpot_error_pct": 31.63, "tpot_ms": 22.275279941891398, "tpot_relative": 1.705908, "ttft_error_pct": 25.42, "ttft_ms": 162.58510137456776, "ttft_relative": 0.835949, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20665.122843974994, "interactivity_tok_s": 44.50668892782373, "output_per_gpu": 173.76902382778812, "total_per_gpu": 349.3878779249944, "tpot_error_pct": 32.77, "tpot_ms": 22.46853280013022, "tpot_relative": 1.720708, "ttft_error_pct": 52.62, "ttft_ms": 103.28369923749904, "ttft_relative": 0.531044, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 15708.29060334354, "interactivity_tok_s": 59.09076255653465, "output_per_gpu": 228.80283828133707, "total_per_gpu": 460.0413604760572, "tpot_ms": 16.923118889238523, "tpot_relative": 1.296024, "ttft_ms": 218.00891533116555, "ttft_relative": 1.120917, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.15, "tpot_shape_error_pct": 5.11, "ttft_mape_pct": 50.62, "ttft_shape_error_pct": 55.67}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.45, "tpot_shape_error_pct": 6.9, "ttft_mape_pct": 46.16, "ttft_shape_error_pct": 58.89}, "gpu": "b300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.15, "tpot_shape_error_pct": 5.11, "ttft_mape_pct": 50.62, "ttft_shape_error_pct": 55.67}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.45, "tpot_shape_error_pct": 6.9, "ttft_mape_pct": 46.16, "ttft_shape_error_pct": 58.89}, "framework": "sglang", "id": "604ab5eb9063f75d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6981.328329629767, "interactivity_tok_s": 150.12612336567494, "output_per_gpu": 18.526713360759143, "total_per_gpu": null, "tpot_error_pct": 5.4, "tpot_ms": 6.661065893003945, "tpot_relative": 0.945956, "ttft_error_pct": 135.54, "ttft_ms": 167.05792108673043, "ttft_relative": 2.355411, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6213.654921600003, "interactivity_tok_s": 150.19413716939528, "output_per_gpu": 18.53578955593863, "total_per_gpu": 36.94081871236174, "tpot_error_pct": 5.45, "tpot_ms": 6.6580495007748395, "tpot_relative": 0.945528, "ttft_error_pct": 20.63, "ttft_ms": 85.55685130000067, "ttft_relative": 1.206298, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6552.090491598938, "interactivity_tok_s": 142.01274949167455, "output_per_gpu": 17.577121964759506, "total_per_gpu": 35.030246433566184, "tpot_ms": 7.041621288084593, "tpot_relative": 1.0, "ttft_ms": 70.92516472330317, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7319.879077236038, "interactivity_tok_s": 143.11780297243712, "output_per_gpu": 35.34351441670827, "total_per_gpu": null, "tpot_error_pct": 8.54, "tpot_ms": 6.987250916593436, "tpot_relative": 0.992279, "ttft_error_pct": 46.61, "ttft_ms": 171.92138956095295, "ttft_relative": 2.423983, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6616.157176750014, "interactivity_tok_s": 143.1088424097384, "output_per_gpu": 35.23140608698185, "total_per_gpu": 69.81864532650324, "tpot_error_pct": 8.53, "tpot_ms": 6.987688413668219, "tpot_relative": 0.992341, "ttft_error_pct": 25.69, "ttft_ms": 87.14335084999986, "ttft_relative": 1.228666, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7254.9415954970755, "interactivity_tok_s": 130.89513234600693, "output_per_gpu": 32.141345200144066, "total_per_gpu": 63.69502185933922, "tpot_ms": 7.639703494524225, "tpot_relative": 1.084935, "ttft_ms": 117.26546640857123, "ttft_relative": 1.653369, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8312.00733225085, "interactivity_tok_s": 125.81081519101096, "output_per_gpu": 61.03911076952508, "total_per_gpu": null, "tpot_error_pct": 4.21, "tpot_ms": 7.948442258177569, "tpot_relative": 1.12878, "ttft_error_pct": 40.18, "ttft_ms": 180.7509021351962, "ttft_relative": 2.548473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7512.806917850001, "interactivity_tok_s": 123.52247943083144, "output_per_gpu": 59.31690946372099, "total_per_gpu": 119.21391564752358, "tpot_error_pct": 2.44, "tpot_ms": 8.09569241653676, "tpot_relative": 1.149692, "ttft_error_pct": 29.97, "ttft_ms": 90.29321140000002, "ttft_relative": 1.273077, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7736.749130277894, "interactivity_tok_s": 120.51080939233725, "output_per_gpu": 57.47627849634714, "total_per_gpu": 115.5146530448937, "tpot_ms": 8.298010817804577, "tpot_relative": 1.178423, "ttft_ms": 128.9377054956276, "ttft_relative": 1.81794, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10220.30593865739, "interactivity_tok_s": 102.07653204234748, "output_per_gpu": 99.64107094409094, "total_per_gpu": null, "tpot_error_pct": 3.16, "tpot_ms": 9.796571062829015, "tpot_relative": 1.391238, "ttft_error_pct": 35.82, "ttft_ms": 198.41374138330733, "ttft_relative": 2.797508, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9326.459144949997, "interactivity_tok_s": 100.58694290714178, "output_per_gpu": 97.76685187212246, "total_per_gpu": 194.81156072998618, "tpot_error_pct": 1.73, "tpot_ms": 9.941648201031061, "tpot_relative": 1.411841, "ttft_error_pct": 34.93, "ttft_ms": 95.0542124499998, "ttft_relative": 1.340204, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 9537.698245575302, "interactivity_tok_s": 98.84595779697992, "output_per_gpu": 95.35431694144555, "total_per_gpu": 190.00431076580077, "tpot_ms": 10.116751582840683, "tpot_relative": 1.436708, "ttft_ms": 146.08170323626837, "ttft_relative": 2.05966, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12773.632422458588, "interactivity_tok_s": 81.57974269923987, "output_per_gpu": 160.09402459354385, "total_per_gpu": null, "tpot_error_pct": 0.76, "tpot_ms": 12.257945010769415, "tpot_relative": 1.740784, "ttft_error_pct": 25.15, "ttft_ms": 233.7546764414761, "ttft_relative": 3.295793, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11536.89396437502, "interactivity_tok_s": 80.0671646594179, "output_per_gpu": 155.7385071149817, "total_per_gpu": 313.1349035259337, "tpot_error_pct": 1.11, "tpot_ms": 12.489514325300577, "tpot_relative": 1.77367, "ttft_error_pct": 66.53, "ttft_ms": 104.53187252499974, "ttft_relative": 1.473833, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 11618.043998289795, "interactivity_tok_s": 80.95956593397254, "output_per_gpu": 154.31201943889602, "total_per_gpu": 310.26674272802495, "tpot_ms": 12.351844880388327, "tpot_relative": 1.754119, "ttft_ms": 312.3147270263871, "ttft_relative": 4.40344, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16694.08990794575, "interactivity_tok_s": 62.41765988823595, "output_per_gpu": 246.1748652289682, "total_per_gpu": null, "tpot_error_pct": 3.62, "tpot_ms": 16.02110687569165, "tpot_relative": 2.275201, "ttft_error_pct": 27.68, "ttft_ms": 304.49757411318774, "ttft_relative": 4.293223, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15216.344121103097, "interactivity_tok_s": 61.16812767229245, "output_per_gpu": 237.91336243057717, "total_per_gpu": 475.07414267937475, "tpot_error_pct": 5.74, "tpot_ms": 16.348383350189966, "tpot_relative": 2.321679, "ttft_error_pct": 70.87, "ttft_ms": 122.67053946875004, "ttft_relative": 1.729577, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 14693.654638734733, "interactivity_tok_s": 64.67836502453262, "output_per_gpu": 245.8091015908695, "total_per_gpu": 490.84064471219097, "tpot_ms": 15.461120571317752, "tpot_relative": 2.195676, "ttft_ms": 421.043746554642, "ttft_relative": 5.936451, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22731.68721926153, "interactivity_tok_s": 45.71405140029616, "output_per_gpu": 362.13070193013857, "total_per_gpu": null, "tpot_error_pct": 3.35, "tpot_ms": 21.875112123479862, "tpot_relative": 3.106545, "ttft_error_pct": 43.33, "ttft_ms": 353.4475169416297, "ttft_relative": 4.983387, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20863.39050425621, "interactivity_tok_s": 44.5210519045715, "output_per_gpu": 346.0653337553443, "total_per_gpu": 692.2867005006302, "tpot_error_pct": 6.12, "tpot_ms": 22.461284206479366, "tpot_relative": 3.189789, "ttft_error_pct": 74.49, "ttft_ms": 159.08156124687477, "ttft_relative": 2.242949, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 20121.520055859946, "interactivity_tok_s": 47.24768763512594, "output_per_gpu": 356.8844572068133, "total_per_gpu": 713.9298255003205, "tpot_ms": 21.16505695945546, "tpot_relative": 3.005708, "ttft_ms": 623.658695521226, "ttft_relative": 8.793193, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.24, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.68, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 50.0, "points": 1, "status_counts": {"failed": 1, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.24, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.68, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "7bd8517a42a772fd", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8602, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 57475.31914377785, "interactivity_tok_s": 17.827865920929046, "output_per_gpu": 8320.918568056726, "total_per_gpu": 9511.275580035046, "tpot_ms": 56.09196324648419, "tpot_relative": 1.0, "ttft_ms": 5844.780718050155, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 116724.659, "interactivity_tok_s": 8.776163938742377, "output_per_gpu": 3546.8638928571427, "total_per_gpu": null, "tpot_error_pct": 28.24, "tpot_ms": 113.945, "tpot_relative": 2.031396, "ttft_error_pct": 97.68, "ttft_ms": 158.924, "ttft_relative": 0.027191, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12288, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 88656.45127235563, "interactivity_tok_s": 11.254677180689256, "output_per_gpu": 7679.920981562409, "total_per_gpu": 8777.894006404835, "tpot_ms": 88.85194874498907, "tpot_relative": 1.584041, "ttft_ms": 6861.262829096378, "ttft_relative": 1.173913, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "5445abdb6a69f7de", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 7048.988902830752, "interactivity_tok_s": 135.8065794933645, "output_per_gpu": 15.888275246371627, "total_per_gpu": 28.383939339164407, "tpot_ms": 7.363413493886428, "tpot_relative": 1.0, "ttft_ms": 296.1392313998658, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 12897.512567135467, "interactivity_tok_s": 105.76095552058433, "output_per_gpu": 50.587362781574434, "total_per_gpu": 90.13289897301031, "tpot_ms": 9.45528522390637, "tpot_relative": 1.28409, "ttft_ms": 4225.801221710087, "ttft_relative": 14.269643, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 84, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 16114.632108935066, "interactivity_tok_s": 75.44086440732194, "output_per_gpu": 141.74718854258197, "total_per_gpu": 252.30290575614916, "tpot_ms": 13.255415454955267, "tpot_relative": 1.800173, "ttft_ms": 3916.6276845551606, "ttft_relative": 13.225629, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "68e76db626098881", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 28913.062792829354, "interactivity_tok_s": 36.32305654025817, "output_per_gpu": 4297.431633909888, "total_per_gpu": 5727.709162592774, "tpot_ms": 27.53072277636282, "tpot_relative": 1.0, "ttft_ms": 3543.546461762335, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "736e47acffc9aa50", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 21470.98894274488, "interactivity_tok_s": 52.58900328136469, "output_per_gpu": 1570.5680726536107, "total_per_gpu": 2512.5148410062384, "tpot_ms": 19.015382258715626, "tpot_relative": 1.0, "ttft_ms": 3958.01123464768, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}]}, {"aic": {"coverage_pct": 94.74, "points": 18, "status_counts": {"failed": 1, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.94, "tpot_shape_error_pct": 8.06, "ttft_mape_pct": 43.8, "ttft_shape_error_pct": 20.82}, "aisimulate": {"coverage_pct": 52.63, "points": 10, "status_counts": {"failed": 9, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.94, "tpot_shape_error_pct": 6.7, "ttft_mape_pct": 53.5, "ttft_shape_error_pct": 15.56}, "gpu": "h200", "precisions": ["fp8"], "rows": 19, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.55, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.24, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "b89648af01683725", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 42304.778, "interactivity_tok_s": 24.440914090186972, "output_per_gpu": 528.1097142857143, "total_per_gpu": null, "tpot_error_pct": 2.55, "tpot_ms": 40.915, "tpot_relative": 1.02547, "ttft_error_pct": 91.24, "ttft_ms": 448.733, "ttft_relative": 0.087621, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1536, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 41874.92695741081, "interactivity_tok_s": 25.06341356184706, "output_per_gpu": 673.8105203431131, "total_per_gpu": 1154.8317037307866, "tpot_ms": 39.898795011795855, "tpot_relative": 1.0, "ttft_ms": 5121.296924551521, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 83.33, "points": 5, "status_counts": {"failed": 1, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.42, "tpot_shape_error_pct": 11.44, "ttft_mape_pct": 55.52, "ttft_shape_error_pct": 47.82}, "aisimulate": {"points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "4c20c61753903c78", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 11, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 11890.02307379957, "interactivity_tok_s": 97.39816034662286, "output_per_gpu": 11.08252034340419, "total_per_gpu": 19.986499736050735, "tpot_ms": 10.26713437339244, "tpot_relative": 1.0, "ttft_ms": 2434.310145949183, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 12059.783, "interactivity_tok_s": 88.1057268722467, "output_per_gpu": 36.4585, "total_per_gpu": null, "tpot_error_pct": 15.19, "tpot_ms": 11.35, "tpot_relative": 1.105469, "ttft_error_pct": 27.96, "ttft_ms": 448.733, "ttft_relative": 0.184337, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 36, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 13006.718201020785, "interactivity_tok_s": 74.72493488529463, "output_per_gpu": 34.46584034619862, "total_per_gpu": 61.857962061037334, "tpot_ms": 13.382413802501597, "tpot_relative": 1.303422, "ttft_ms": 622.8887425593308, "ttft_relative": 0.255879, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 16238.738, "interactivity_tok_s": 64.7878198898607, "output_per_gpu": 53.7338875, "total_per_gpu": null, "tpot_error_pct": 5.06, "tpot_ms": 15.435, "tpot_relative": 1.503341, "ttft_error_pct": 49.07, "ttft_ms": 448.733, "ttft_relative": 0.184337, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 72, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 15830.803944427884, "interactivity_tok_s": 61.51145680753663, "output_per_gpu": 56.08616768155505, "total_per_gpu": 101.12471244200749, "tpot_ms": 16.25713406738037, "tpot_relative": 1.583415, "ttft_ms": 881.1532965382664, "ttft_relative": 0.361972, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 19288.301, "interactivity_tok_s": 54.300608166811465, "output_per_gpu": 89.9804125, "total_per_gpu": null, "tpot_error_pct": 8.65, "tpot_ms": 18.416, "tpot_relative": 1.793685, "ttft_error_pct": 54.13, "ttft_ms": 448.733, "ttft_relative": 0.184337, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 144, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 19501.840108128254, "interactivity_tok_s": 49.606172532018654, "output_per_gpu": 91.32825342222347, "total_per_gpu": 164.58097067678148, "tpot_ms": 20.158781638606424, "tpot_relative": 1.963428, "ttft_ms": 978.2979881643163, "ttft_relative": 0.401879, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 25072.343, "interactivity_tok_s": 41.54549231408392, "output_per_gpu": 137.7792, "total_per_gpu": null, "tpot_error_pct": 4.44, "tpot_ms": 24.07, "tpot_relative": 2.344374, "ttft_error_pct": 73.48, "ttft_ms": 448.733, "ttft_relative": 0.184337, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 288, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 24884.21267038171, "interactivity_tok_s": 39.699072420798466, "output_per_gpu": 143.19093282562392, "total_per_gpu": 257.6423127648354, "tpot_ms": 25.189505422199662, "tpot_relative": 2.453411, "ttft_ms": 1691.9208172331207, "ttft_relative": 0.695031, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 30999.605, "interactivity_tok_s": 33.4851326011251, "output_per_gpu": 222.03647500000002, "total_per_gpu": null, "tpot_error_pct": 3.78, "tpot_ms": 29.864, "tpot_relative": 2.908699, "ttft_error_pct": 72.94, "ttft_ms": 448.733, "ttft_relative": 0.184337, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 576, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 30249.67193942648, "interactivity_tok_s": 32.21811319253903, "output_per_gpu": 235.82998398328508, "total_per_gpu": 424.3142335220966, "tpot_ms": 31.038440830593917, "tpot_relative": 3.023087, "ttft_ms": 1658.5354576794796, "ttft_relative": 0.681316, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 80}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.91, "tpot_shape_error_pct": null, "ttft_mape_pct": 86.35, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "e7db74021675af72", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 35746.325, "interactivity_tok_s": 28.982146997449572, "output_per_gpu": 369.6768, "total_per_gpu": null, "tpot_error_pct": 0.91, "tpot_ms": 34.504, "tpot_relative": 0.990907, "ttft_error_pct": 86.35, "ttft_ms": 448.733, "ttft_relative": 0.136473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1152, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 35358.35951628087, "interactivity_tok_s": 28.718621225649724, "output_per_gpu": 399.04166785990475, "total_per_gpu": 718.1625470898305, "tpot_ms": 34.820613153491536, "tpot_relative": 1.0, "ttft_ms": 3288.0777473435955, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 80}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.76, "tpot_shape_error_pct": null, "ttft_mape_pct": 90.54, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "0bbfc0372ba1dc47", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 53018.657, "interactivity_tok_s": 19.45979606133728, "output_per_gpu": 821.504, "total_per_gpu": null, "tpot_error_pct": 5.76, "tpot_ms": 51.388, "tpot_relative": 1.057637, "ttft_error_pct": 90.54, "ttft_ms": 448.733, "ttft_relative": 0.09463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 3584, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 49463.7985717979, "interactivity_tok_s": 20.581409937441272, "output_per_gpu": 1142.0525714691883, "total_per_gpu": 1777.1845848540606, "tpot_ms": 48.587536181416844, "tpot_relative": 1.0, "ttft_ms": 4741.952200496365, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.93, "tpot_shape_error_pct": 4.96, "ttft_mape_pct": 23.42, "ttft_shape_error_pct": 8.1}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.44, "tpot_shape_error_pct": 4.44, "ttft_mape_pct": 56.08, "ttft_shape_error_pct": 10.03}, "framework": "sglang", "id": "d13e456d75e5f4bb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10508.734556814818, "interactivity_tok_s": 98.51499376706317, "output_per_gpu": 48.50115140383629, "total_per_gpu": null, "tpot_error_pct": 4.65, "tpot_ms": 10.150739108449633, "tpot_relative": 1.046504, "ttft_error_pct": 21.07, "ttft_ms": 124.52844887084274, "ttft_relative": 0.78932, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9433.41342209999, "interactivity_tok_s": 97.87313255402269, "output_per_gpu": 47.307166532086214, "total_per_gpu": 95.07697908517598, "tpot_error_pct": 5.34, "tpot_ms": 10.217308610695929, "tpot_relative": 1.053367, "ttft_error_pct": 58.01, "ttft_ms": 66.24156674999989, "ttft_relative": 0.41987, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 9051.46253653802, "interactivity_tok_s": 103.09633817958112, "output_per_gpu": 49.173854925265026, "total_per_gpu": 98.82861137108823, "tpot_ms": 9.699665552215086, "tpot_relative": 1.0, "ttft_ms": 157.76667634490877, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12518.901249716902, "interactivity_tok_s": 82.62051520932509, "output_per_gpu": 81.57662456019915, "total_per_gpu": null, "tpot_error_pct": 3.19, "tpot_ms": 12.103531398544625, "tpot_relative": 1.24783, "ttft_error_pct": 17.16, "ttft_ms": 136.98862900575088, "ttft_relative": 0.868299, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11389.780763750001, "interactivity_tok_s": 82.06356966706252, "output_per_gpu": 80.0114382118129, "total_per_gpu": 159.43188162263712, "tpot_error_pct": 3.89, "tpot_ms": 12.18567513035404, "tpot_relative": 1.256298, "ttft_error_pct": 53.76, "ttft_ms": 76.45832160000059, "ttft_relative": 0.484629, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 11054.064605961321, "interactivity_tok_s": 85.25607543075749, "output_per_gpu": 82.37151486488753, "total_per_gpu": 164.13460250833404, "tpot_ms": 11.729369372768875, "tpot_relative": 1.209255, "ttft_ms": 165.3588728122486, "ttft_relative": 1.048123, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18673.509086461483, "interactivity_tok_s": 55.26314243736969, "output_per_gpu": 109.65379598433128, "total_per_gpu": null, "tpot_error_pct": 18.39, "tpot_ms": 18.09524315656336, "tpot_relative": 1.865553, "ttft_error_pct": 18.08, "ttft_ms": 162.0753372971679, "ttft_relative": 1.02731, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16750.17698078744, "interactivity_tok_s": 54.93587928342671, "output_per_gpu": 107.47630246486956, "total_per_gpu": 216.0967266676945, "tpot_error_pct": 19.09, "tpot_ms": 18.20303985380433, "tpot_relative": 1.876667, "ttft_error_pct": 54.4, "ttft_ms": 90.21455409374974, "ttft_relative": 0.571823, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 14186.768996267347, "interactivity_tok_s": 65.42399886454702, "output_per_gpu": 126.61789679945585, "total_per_gpu": 254.58368410891322, "tpot_ms": 15.284911001395479, "tpot_relative": 1.575818, "ttft_ms": 197.84106940496713, "ttft_relative": 1.254011, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24074.21034834811, "interactivity_tok_s": 42.872777367430466, "output_per_gpu": 170.50149779641137, "total_per_gpu": null, "tpot_error_pct": 7.0, "tpot_ms": 23.324824315200036, "tpot_relative": 2.404704, "ttft_error_pct": 30.65, "ttft_ms": 212.91507389847507, "ttft_relative": 1.349557, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21985.044849078164, "interactivity_tok_s": 42.204215523969985, "output_per_gpu": 164.80913945392828, "total_per_gpu": 329.09694450074164, "tpot_error_pct": 8.7, "tpot_ms": 23.694315546086802, "tpot_relative": 2.442797, "ttft_error_pct": 61.6, "ttft_ms": 117.90710014999979, "ttft_relative": 0.747351, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 20428.376156761078, "interactivity_tok_s": 45.87429314017432, "output_per_gpu": 177.14051522562795, "total_per_gpu": 353.7206886778132, "tpot_ms": 21.79870100547123, "tpot_relative": 2.247366, "ttft_ms": 307.0151459571207, "ttft_relative": 1.946008, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31135.463410520857, "interactivity_tok_s": 33.123773132767695, "output_per_gpu": 263.8956450369459, "total_per_gpu": null, "tpot_error_pct": 1.42, "tpot_ms": 30.189797399945054, "tpot_relative": 3.112458, "ttft_error_pct": 30.15, "ttft_ms": 251.30067037706695, "ttft_relative": 1.592863, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29014.406757987384, "interactivity_tok_s": 31.9406255096048, "output_per_gpu": 248.76726254830496, "total_per_gpu": 497.6466885986683, "tpot_error_pct": 5.18, "tpot_ms": 31.308090685302705, "tpot_relative": 3.22775, "ttft_error_pct": 52.62, "ttft_ms": 170.45523330624943, "ttft_relative": 1.080426, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 27786.0724129976, "interactivity_tok_s": 33.593936842967736, "output_per_gpu": 259.7972132035421, "total_per_gpu": 519.7115630630872, "tpot_ms": 29.767276299721072, "tpot_relative": 3.068897, "ttft_ms": 359.7531621941016, "ttft_relative": 2.280286, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.8, "tpot_shape_error_pct": 7.78, "ttft_mape_pct": 25.13, "ttft_shape_error_pct": 6.55}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.45, "tpot_shape_error_pct": 8.96, "ttft_mape_pct": 50.92, "ttft_shape_error_pct": 21.09}, "framework": "trt", "id": "f63b9f751030f8f0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10767.1346316825, "interactivity_tok_s": 96.19202792104902, "output_per_gpu": 47.32786464374978, "total_per_gpu": null, "tpot_error_pct": 12.36, "tpot_ms": 10.395871899288414, "tpot_relative": 0.876358, "ttft_error_pct": 28.18, "ttft_ms": 132.1576787104534, "ttft_relative": 0.718163, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9638.026938200055, "interactivity_tok_s": 95.89195775411008, "output_per_gpu": 46.43703133924001, "total_per_gpu": 93.3281991096219, "tpot_error_pct": 12.09, "tpot_ms": 10.428403209414487, "tpot_relative": 0.8791, "ttft_error_pct": 58.01, "ttft_ms": 77.2736617500008, "ttft_relative": 0.419916, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 11058.394283335656, "interactivity_tok_s": 84.29863768376657, "output_per_gpu": 40.34072841002193, "total_per_gpu": 81.07597373685441, "tpot_ms": 11.86258790742677, "tpot_relative": 1.0, "ttft_ms": 184.02190948836505, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14845.46053276071, "interactivity_tok_s": 69.5910357868526, "output_per_gpu": 68.81867230552726, "total_per_gpu": null, "tpot_error_pct": 3.06, "tpot_ms": 14.369666849949713, "tpot_relative": 1.211343, "ttft_error_pct": 29.96, "ttft_ms": 145.29134526215444, "ttft_relative": 0.789533, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13475.896653500113, "interactivity_tok_s": 69.37792384645738, "output_per_gpu": 67.88118754205026, "total_per_gpu": 135.26097891101725, "tpot_error_pct": 2.76, "tpot_ms": 14.413806936816584, "tpot_relative": 1.215064, "ttft_error_pct": 55.38, "ttft_ms": 92.55719303750034, "ttft_relative": 0.502968, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 13972.030363755766, "interactivity_tok_s": 67.46320803759218, "output_per_gpu": 65.26024460389486, "total_per_gpu": 130.0384523123892, "tpot_ms": 14.822894272130894, "tpot_relative": 1.24955, "ttft_ms": 207.42877998854965, "ttft_relative": 1.127196, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17953.42380904554, "interactivity_tok_s": 57.53093807631187, "output_per_gpu": 114.0383331480922, "total_per_gpu": null, "tpot_error_pct": 17.11, "tpot_ms": 17.381951927735834, "tpot_relative": 1.465275, "ttft_error_pct": 26.6, "ttft_ms": 171.6869869717807, "ttft_relative": 0.93297, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16183.732834624961, "interactivity_tok_s": 56.9338932719171, "output_per_gpu": 111.00392228370127, "total_per_gpu": 223.18951901628495, "tpot_error_pct": 16.24, "tpot_ms": 17.564230066332993, "tpot_relative": 1.480641, "ttft_error_pct": 53.19, "ttft_ms": 109.48030448749996, "ttft_relative": 0.594931, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 19433.434604288777, "interactivity_tok_s": 47.68959335749685, "output_per_gpu": 92.58517349066831, "total_per_gpu": 186.15594759443616, "tpot_ms": 20.96893535039546, "tpot_relative": 1.767653, "ttft_ms": 233.8934774743393, "ttft_relative": 1.271009, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23838.44299422543, "interactivity_tok_s": 43.322765300123905, "output_per_gpu": 172.2009788772855, "total_per_gpu": null, "tpot_error_pct": 7.79, "tpot_ms": 23.0825523964681, "tpot_relative": 1.945828, "ttft_error_pct": 20.6, "ttft_ms": 224.9918926385622, "ttft_relative": 1.222636, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21829.186194215443, "interactivity_tok_s": 42.55569757313393, "output_per_gpu": 165.91905650004324, "total_per_gpu": 331.31326763510174, "tpot_error_pct": 6.13, "tpot_ms": 23.498616096738957, "tpot_relative": 1.980901, "ttft_error_pct": 49.78, "ttft_ms": 142.30922726249977, "ttft_relative": 0.773328, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 23396.726763737388, "interactivity_tok_s": 39.94709646651886, "output_per_gpu": 154.54931245869548, "total_per_gpu": 308.60974502609594, "tpot_ms": 25.03310849733815, "tpot_relative": 2.110257, "ttft_ms": 283.37134614703245, "ttft_relative": 1.539878, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30229.418307683878, "interactivity_tok_s": 34.13967745466719, "output_per_gpu": 271.8506585485912, "total_per_gpu": null, "tpot_error_pct": 3.7, "tpot_ms": 29.291430808854678, "tpot_relative": 2.469228, "ttft_error_pct": 20.31, "ttft_ms": 264.2845902255426, "ttft_relative": 1.436158, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28221.233476956422, "interactivity_tok_s": 32.88536977276913, "output_per_gpu": 255.6980574777114, "total_per_gpu": 511.5114033953181, "tpot_error_pct": 0.03, "tpot_ms": 30.408659136563948, "tpot_relative": 2.563409, "ttft_error_pct": 38.26, "ttft_ms": 204.74611793281207, "ttft_relative": 1.112618, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 28360.017326825935, "interactivity_tok_s": 32.87632227440298, "output_per_gpu": 253.75737409712272, "total_per_gpu": 507.6291616241329, "tpot_ms": 30.417027538952713, "tpot_relative": 2.564114, "ttft_ms": 331.6437606518775, "ttft_relative": 1.802197, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 54}, {"aic": {"coverage_pct": 82.81, "points": 53, "status_counts": {"failed": 11, "success": 53, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.49, "tpot_shape_error_pct": 6.47, "ttft_mape_pct": 32.83, "ttft_shape_error_pct": 30.43}, "aisimulate": {"coverage_pct": 59.38, "points": 38, "status_counts": {"failed": 26, "success": 38, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.68, "tpot_shape_error_pct": 6.52, "ttft_mape_pct": 58.11, "ttft_shape_error_pct": 67.1}, "gpu_skus": ["b200", "b300", "gb300", "h200"], "gpus": [{"aic": {"coverage_pct": 89.66, "points": 26, "status_counts": {"failed": 3, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.68, "tpot_shape_error_pct": 5.76, "ttft_mape_pct": 25.44, "ttft_shape_error_pct": 31.69}, "aisimulate": {"coverage_pct": 65.52, "points": 19, "status_counts": {"failed": 10, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.5, "tpot_shape_error_pct": 7.87, "ttft_mape_pct": 81.68, "ttft_shape_error_pct": 119.84}, "gpu": "b200", "precisions": ["fp8"], "rows": 29, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 77.06, "tpot_shape_error_pct": null, "ttft_mape_pct": 24.9, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "0c4ab56607d1565e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 24853.909, "interactivity_tok_s": 44.57917261055635, "output_per_gpu": 70.35319642857142, "total_per_gpu": null, "tpot_error_pct": 77.06, "tpot_ms": 22.432, "tpot_relative": 1.770627, "ttft_error_pct": 24.9, "ttft_ms": 1905.973, "ttft_relative": 0.751018, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 96, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 14197.713966896603, "interactivity_tok_s": 78.93309919068078, "output_per_gpu": 124.854649565955, "total_per_gpu": 964.3756269421885, "tpot_ms": 12.668956499278883, "tpot_relative": 1.0, "ttft_ms": 2537.8537762517226, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 56.83, "tpot_shape_error_pct": null, "ttft_mape_pct": 8.88, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "e57e35f6c7d9f695", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12103.563, "interactivity_tok_s": 87.49671887304225, "output_per_gpu": 5.0636875, "total_per_gpu": null, "tpot_error_pct": 56.83, "tpot_ms": 11.429, "tpot_relative": 1.568299, "ttft_error_pct": 8.88, "ttft_ms": 411.696, "ttft_relative": 1.088826, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 7150.579427718185, "interactivity_tok_s": 137.2210180570491, "output_per_gpu": 16.261908471041686, "total_per_gpu": 72.7931478511866, "tpot_ms": 7.287513342775623, "tpot_relative": 1.0, "ttft_ms": 378.10992603190243, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.03, "tpot_shape_error_pct": null, "ttft_mape_pct": 3.92, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "77eecc3854af04a7", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 28041.577, "interactivity_tok_s": 39.14200720212933, "output_per_gpu": 192.26283333333333, "total_per_gpu": null, "tpot_error_pct": 2.03, "tpot_ms": 25.548, "tpot_relative": 0.979714, "ttft_error_pct": 3.92, "ttft_ms": 1905.973, "ttft_relative": 0.960763, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 25949.092336738933, "interactivity_tok_s": 38.347983675981176, "output_per_gpu": 271.35664513675374, "total_per_gpu": 1633.4141469464864, "tpot_ms": 26.076990343206457, "tpot_relative": 1.0, "ttft_ms": 1983.8114366673835, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.72, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "dae7e6e874c1dd81", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 43059.217, "interactivity_tok_s": 24.85830764641543, "output_per_gpu": 580.224, "total_per_gpu": null, "tpot_error_pct": 5.5, "tpot_ms": 40.228, "tpot_relative": 0.944969, "ttft_error_pct": 53.72, "ttft_ms": 1905.973, "ttft_relative": 0.462778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 640, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 43304.23800878649, "interactivity_tok_s": 23.490336452674548, "output_per_gpu": 1636.73538298405, "total_per_gpu": 4914.464633589062, "tpot_ms": 42.57069718923258, "tpot_relative": 1.0, "ttft_ms": 4118.543900192672, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 76.4, "tpot_shape_error_pct": 4.42, "ttft_mape_pct": 34.19, "ttft_shape_error_pct": 52.41}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "6a16686dd103fa35", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 20882.623, "interactivity_tok_s": 53.908355795148246, "output_per_gpu": 39.755775, "total_per_gpu": null, "tpot_error_pct": 80.38, "tpot_ms": 18.55, "tpot_relative": 1.803848, "ttft_error_pct": 30.47, "ttft_ms": 1905.973, "ttft_relative": 1.304697, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 10967.312451032922, "interactivity_tok_s": 97.24249452293958, "output_per_gpu": 80.92116380837463, "total_per_gpu": 578.9682021876605, "tpot_ms": 10.283570006157126, "tpot_relative": 1.0, "ttft_ms": 1460.8548465526837, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 30486.547, "interactivity_tok_s": 35.793542844870785, "output_per_gpu": 105.51295, "total_per_gpu": null, "tpot_error_pct": 72.41, "tpot_ms": 27.938, "tpot_relative": 2.716761, "ttft_error_pct": 37.91, "ttft_ms": 1905.973, "ttft_relative": 1.304697, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 17961.815181339443, "interactivity_tok_s": 61.710177980197244, "output_per_gpu": 197.2577449664807, "total_per_gpu": 1424.8587615869237, "tpot_ms": 16.204782302214383, "tpot_relative": 1.575793, "ttft_ms": 3069.659323356609, "ttft_relative": 2.101276, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.92, "tpot_shape_error_pct": 3.47, "ttft_mape_pct": 43.69, "ttft_shape_error_pct": 46.4}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "16a6c58e00a77c37", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 29107.741, "interactivity_tok_s": 36.97404422095689, "output_per_gpu": 272.378875, "total_per_gpu": null, "tpot_error_pct": 13.43, "tpot_ms": 27.046, "tpot_relative": 0.86572, "ttft_error_pct": 26.68, "ttft_ms": 1439.683, "ttft_relative": 0.733233, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "21957354903", "measured": {"e2e_ms": 30675.890215706477, "interactivity_tok_s": 32.00916544258389, "output_per_gpu": 461.8214647545772, "total_per_gpu": 2084.9303775494654, "tpot_ms": 31.24105193538206, "tpot_relative": 1.0, "ttft_ms": 1963.4728290709518, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 32356.789, "interactivity_tok_s": 33.08847859175435, "output_per_gpu": 487.526375, "total_per_gpu": null, "tpot_error_pct": 10.42, "tpot_ms": 30.222, "tpot_relative": 0.967381, "ttft_error_pct": 60.7, "ttft_ms": 1439.683, "ttft_relative": 0.733233, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "21957354903", "measured": {"e2e_ms": 34736.61172833977, "interactivity_tok_s": 29.64008632699306, "output_per_gpu": 815.1832243519142, "total_per_gpu": 3667.580089284787, "tpot_ms": 33.738093370170304, "tpot_relative": 1.079928, "ttft_ms": 3663.2349655162216, "ttft_relative": 1.865692, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.95, "tpot_shape_error_pct": 1.75, "ttft_mape_pct": 24.85, "ttft_shape_error_pct": 46.41}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "88ece11c5a10518a", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 29574.031, "interactivity_tok_s": 36.97404422095689, "output_per_gpu": 272.378875, "total_per_gpu": null, "tpot_error_pct": 11.72, "tpot_ms": 27.046, "tpot_relative": 0.882786, "ttft_error_pct": 2.15, "ttft_ms": 1905.973, "ttft_relative": 0.978534, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 30105.604339955153, "interactivity_tok_s": 32.640164015641105, "output_per_gpu": 470.19326307109196, "total_per_gpu": 2122.7255385736394, "tpot_ms": 30.637100950865378, "tpot_relative": 1.0, "ttft_ms": 1947.784445944626, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 32823.079, "interactivity_tok_s": 33.08847859175435, "output_per_gpu": 435.168, "total_per_gpu": null, "tpot_error_pct": 10.18, "tpot_ms": 30.222, "tpot_relative": 0.986451, "ttft_error_pct": 47.56, "ttft_ms": 1905.973, "ttft_relative": 0.978534, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 34623.079832746065, "interactivity_tok_s": 29.721005118563397, "output_per_gpu": 816.990931758649, "total_per_gpu": 3675.7075851115537, "tpot_ms": 33.64623760235523, "tpot_relative": 1.098219, "ttft_ms": 3634.54978417044, "ttft_relative": 1.865992, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 66.67, "points": 4, "status_counts": {"failed": 2, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.87, "tpot_shape_error_pct": 1.17, "ttft_mape_pct": 11.23, "ttft_shape_error_pct": 21.51}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.91, "tpot_shape_error_pct": 3.51, "ttft_mape_pct": 52.97, "ttft_shape_error_pct": 19.24}, "framework": "sglang", "id": "a46c697139024a05", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8218.081827089325, "interactivity_tok_s": 132.15404178373763, "output_per_gpu": 32.00043125426034, "total_per_gpu": null, "tpot_error_pct": 4.95, "tpot_ms": 7.5669270988808774, "tpot_relative": 0.950471, "ttft_error_pct": 16.32, "ttft_ms": 477.1154049341884, "ttft_relative": 1.163182, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7238.090702699992, "interactivity_tok_s": 132.60077055781596, "output_per_gpu": 32.13209250241139, "total_per_gpu": 287.63110128246916, "tpot_error_pct": 5.27, "tpot_ms": 7.5414343053457955, "tpot_relative": 0.947269, "ttft_error_pct": 43.99, "ttft_ms": 229.75498550000083, "ttft_relative": 0.56013, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7808.671367634088, "interactivity_tok_s": 125.60862848070609, "output_per_gpu": 29.78239529668508, "total_per_gpu": 266.5977373671662, "tpot_ms": 7.9612365177094775, "tpot_relative": 1.0, "ttft_ms": 410.1812979206443, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9481.127720424649, "interactivity_tok_s": 113.77538699200039, "output_per_gpu": 55.34191569545552, "total_per_gpu": null, "tpot_error_pct": 3.75, "tpot_ms": 8.789247186391117, "tpot_relative": 1.104005, "ttft_error_pct": 1.26, "ttft_ms": 489.7278487465363, "ttft_relative": 1.19393, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8426.406487300012, "interactivity_tok_s": 114.16693749641905, "output_per_gpu": 55.343352205363, "total_per_gpu": 493.1904857396286, "tpot_error_pct": 4.08, "tpot_ms": 8.759103308971268, "tpot_relative": 1.100219, "ttft_error_pct": 50.08, "ttft_ms": 241.43805339999963, "ttft_relative": 0.588613, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 9018.87229445856, "interactivity_tok_s": 109.50510562299328, "output_per_gpu": 51.72033962386891, "total_per_gpu": 460.9041267876549, "tpot_ms": 9.1319942966205, "tpot_relative": 1.147057, "ttft_ms": 483.6424052948132, "ttft_relative": 1.179094, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11660.127893375182, "interactivity_tok_s": 91.78866977723689, "output_per_gpu": 86.13301126158743, "total_per_gpu": null, "tpot_error_pct": 4.32, "tpot_ms": 10.89459082942277, "tpot_relative": 1.368455, "ttft_error_pct": 7.18, "ttft_ms": 514.9614748756893, "ttft_relative": 1.255448, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10626.500045199984, "interactivity_tok_s": 88.7048756295241, "output_per_gpu": 84.81964529748205, "total_per_gpu": 762.4780259448016, "tpot_error_pct": 0.99, "tpot_ms": 11.273337490224325, "tpot_relative": 1.416028, "ttft_error_pct": 52.3, "ttft_ms": 264.64627337500013, "ttft_relative": 0.645193, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 11016.585060197394, "interactivity_tok_s": 87.82637099420413, "output_per_gpu": 81.70419049309088, "total_per_gpu": 734.4719452680772, "tpot_ms": 11.386101790155857, "tpot_relative": 1.430193, "ttft_ms": 554.8170282156207, "ttft_relative": 1.352614, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14986.26457734285, "interactivity_tok_s": 70.93919453923449, "output_per_gpu": 135.0801827494741, "total_per_gpu": null, "tpot_error_pct": 6.44, "tpot_ms": 14.096579563599752, "tpot_relative": 1.770652, "ttft_error_pct": 20.18, "ttft_ms": 565.4636837803052, "ttft_relative": 1.37857, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13664.313781400002, "interactivity_tok_s": 69.38367790802877, "output_per_gpu": 133.61102285983995, "total_per_gpu": 1188.5318219477192, "tpot_error_pct": 4.34, "tpot_ms": 14.412611584608495, "tpot_relative": 1.810348, "ttft_error_pct": 55.22, "ttft_ms": 317.2189395625004, "ttft_relative": 0.773363, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 14674.066585942637, "interactivity_tok_s": 66.37126836063548, "output_per_gpu": 124.4021536414562, "total_per_gpu": 1106.6146726292427, "tpot_ms": 15.066760432637682, "tpot_relative": 1.892515, "ttft_ms": 708.4138450038154, "ttft_relative": 1.727075, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 18865.078846825018, "interactivity_tok_s": 49.59537503614087, "output_per_gpu": 189.54308144947046, "total_per_gpu": 1710.5788680019555, "tpot_error_pct": 6.96, "tpot_ms": 20.16317044222945, "tpot_relative": 2.532668, "ttft_error_pct": 56.38, "ttft_ms": 416.4769265000001, "ttft_relative": 1.015348, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 20793.87632718426, "interactivity_tok_s": 46.14229020943399, "output_per_gpu": 172.00100698877137, "total_per_gpu": 1552.2660367241326, "tpot_ms": 21.67209289918483, "tpot_relative": 2.722202, "ttft_ms": 954.6972843934782, "ttft_relative": 2.327501, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26984.599909006334, "interactivity_tok_s": 35.08667320468865, "output_per_gpu": 268.97434309032474, "total_per_gpu": 2405.2542192235283, "tpot_error_pct": 13.8, "tpot_ms": 28.500849714824756, "tpot_relative": 3.579953, "ttft_error_pct": 59.85, "ttft_ms": 605.3450799874995, "ttft_relative": 1.475799, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 32127.947878755913, "interactivity_tok_s": 30.243440120940676, "output_per_gpu": 225.21934600397282, "total_per_gpu": 2013.983103380633, "tpot_ms": 33.06502157165633, "tpot_relative": 4.153252, "ttft_ms": 1507.864161184989, "ttft_relative": 3.676092, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.32, "tpot_shape_error_pct": 6.99, "ttft_mape_pct": 10.0, "ttft_shape_error_pct": 18.46}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.41, "tpot_shape_error_pct": 5.25, "ttft_mape_pct": 50.36, "ttft_shape_error_pct": 14.63}, "framework": "sglang", "id": "4e101f26f0a9ba3e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7861.016722546464, "interactivity_tok_s": 136.17722672310597, "output_per_gpu": 16.616161815932863, "total_per_gpu": null, "tpot_error_pct": 0.69, "tpot_ms": 7.343371752116348, "tpot_relative": 0.993068, "ttft_error_pct": 10.97, "ttft_ms": 348.74742013143964, "ttft_relative": 1.109666, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6979.762862200003, "interactivity_tok_s": 136.52473744666398, "output_per_gpu": 16.660666314291728, "total_per_gpu": 149.13830463172715, "tpot_error_pct": 0.95, "tpot_ms": 7.324679898327358, "tpot_relative": 0.99054, "ttft_error_pct": 45.0, "ttft_ms": 172.8613843999994, "ttft_relative": 0.550021, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7186.234066000907, "interactivity_tok_s": 135.23319120282324, "output_per_gpu": 16.180803782258934, "total_per_gpu": 144.84280509205578, "tpot_ms": 7.394634343133975, "tpot_relative": 1.0, "ttft_ms": 314.28152349981247, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8210.839493959578, "interactivity_tok_s": 130.27071866687405, "output_per_gpu": 31.823761855770957, "total_per_gpu": null, "tpot_error_pct": 6.96, "tpot_ms": 7.676322125443877, "tpot_relative": 1.038094, "ttft_error_pct": 3.53, "ttft_ms": 357.96195963049115, "ttft_relative": 1.138985, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7338.214120550003, "interactivity_tok_s": 130.5747926890904, "output_per_gpu": 31.76527199075219, "total_per_gpu": 283.0751896747657, "tpot_error_pct": 7.18, "tpot_ms": 7.658446009415343, "tpot_relative": 1.035676, "ttft_error_pct": 50.96, "ttft_ms": 181.9589390999999, "ttft_relative": 0.578968, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8080.968671347364, "interactivity_tok_s": 121.20497328665303, "output_per_gpu": 28.87183767620709, "total_per_gpu": 257.29044375350355, "tpot_ms": 8.250486534368298, "tpot_relative": 1.11574, "ttft_ms": 371.06426789832767, "ttft_relative": 1.180675, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9275.287444700292, "interactivity_tok_s": 114.9581484583592, "output_per_gpu": 54.22845410690728, "total_per_gpu": null, "tpot_error_pct": 8.3, "tpot_ms": 8.698817903823718, "tpot_relative": 1.176369, "ttft_error_pct": 15.49, "ttft_ms": 376.3967290886293, "ttft_relative": 1.197642, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8472.909898124975, "interactivity_tok_s": 111.102772996675, "output_per_gpu": 53.057691972026774, "total_per_gpu": 476.9570079447064, "tpot_error_pct": 5.12, "tpot_ms": 9.000675437956236, "tpot_relative": 1.21719, "ttft_error_pct": 55.12, "ttft_ms": 199.8866752500001, "ttft_relative": 0.636012, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 9164.125723198958, "interactivity_tok_s": 105.4157544254718, "output_per_gpu": 49.023793056967776, "total_per_gpu": 440.69466246061876, "tpot_ms": 9.486248098779134, "tpot_relative": 1.282856, "ttft_ms": 445.4112273251667, "ttft_relative": 1.417236, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.93, "tpot_shape_error_pct": 10.84, "ttft_mape_pct": 39.96, "ttft_shape_error_pct": 44.18}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.81, "tpot_shape_error_pct": 9.17, "ttft_mape_pct": 10.76, "ttft_shape_error_pct": 35.0}, "framework": "trt", "id": "3d740f73aa12ca98", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 13910.789898364663, "interactivity_tok_s": 75.82782656230265, "output_per_gpu": 36.33093850319413, "total_per_gpu": null, "tpot_error_pct": 41.93, "tpot_ms": 13.187770840014345, "tpot_relative": 1.419322, "ttft_error_pct": 30.06, "ttft_ms": 419.70032902998935, "ttft_relative": 1.300644, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12667.498422524997, "interactivity_tok_s": 73.8849815805788, "output_per_gpu": 35.48313484260135, "total_per_gpu": 318.9722205773365, "tpot_error_pct": 45.66, "tpot_ms": 13.534550305184855, "tpot_relative": 1.456644, "ttft_error_pct": 29.53, "ttft_ms": 227.3871144000005, "ttft_relative": 0.704669, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 8855.891597046866, "interactivity_tok_s": 107.624117071641, "output_per_gpu": 50.646108316907885, "total_per_gpu": 455.27830912078684, "tpot_ms": 9.291597712567906, "tpot_relative": 1.0, "ttft_ms": 322.6864557269437, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17637.749058581303, "interactivity_tok_s": 59.556706348376366, "output_per_gpu": 57.60854699622829, "total_per_gpu": null, "tpot_error_pct": 45.19, "tpot_ms": 16.790720328799075, "tpot_relative": 1.807086, "ttft_error_pct": 36.34, "ttft_ms": 460.8421622198487, "ttft_relative": 1.428142, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16091.626612837495, "interactivity_tok_s": 58.53839373081138, "output_per_gpu": 56.644629505488595, "total_per_gpu": 503.8801684823248, "tpot_error_pct": 47.72, "tpot_ms": 17.08280559590509, "tpot_relative": 1.838522, "ttft_error_pct": 18.04, "ttft_ms": 277.0065693624989, "ttft_relative": 0.858439, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 11047.52717995143, "interactivity_tok_s": 86.47334106461102, "output_per_gpu": 82.47378671498197, "total_per_gpu": 733.6424636918804, "tpot_ms": 11.564257697095588, "tpot_relative": 1.244593, "ttft_ms": 337.99734528001864, "ttft_relative": 1.047448, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 43061.56004838072, "interactivity_tok_s": 24.2202303156022, "output_per_gpu": 191.33588370977105, "total_per_gpu": null, "tpot_error_pct": 33.2, "tpot_ms": 41.28779895853506, "tpot_relative": 4.443563, "ttft_error_pct": 6.2, "ttft_ms": 824.1417137993477, "ttft_relative": 2.554002, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 40042.52523120168, "interactivity_tok_s": 23.542489217650683, "output_per_gpu": 180.95137305042215, "total_per_gpu": 1631.055501114208, "tpot_error_pct": 37.04, "tpot_ms": 42.476391971765786, "tpot_relative": 4.571484, "ttft_error_pct": 0.89, "ttft_ms": 886.4512140921846, "ttft_relative": 2.747098, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 29461.211541605007, "interactivity_tok_s": 32.26180434790426, "output_per_gpu": 245.78060990435867, "total_per_gpu": 2215.4118484638607, "tpot_ms": 30.99640643828281, "tpot_relative": 3.335961, "ttft_ms": 878.5957236948889, "ttft_relative": 2.722754, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 59453.93664836794, "interactivity_tok_s": 17.450123066396166, "output_per_gpu": 276.8049187014405, "total_per_gpu": null, "tpot_error_pct": 21.72, "tpot_ms": 57.306186105111635, "tpot_relative": 6.167528, "ttft_error_pct": 50.45, "ttft_ms": 829.7082628387342, "ttft_relative": 2.571252, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56930.96215440521, "interactivity_tok_s": 16.63878081492487, "output_per_gpu": 254.4542405440111, "total_per_gpu": 2297.2337094812297, "tpot_error_pct": 27.66, "tpot_ms": 60.1005573138512, "tpot_relative": 6.468269, "ttft_error_pct": 3.81, "ttft_ms": 1610.4880945468751, "ttft_relative": 4.990876, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 45013.67262586973, "interactivity_tok_s": 21.240334634574808, "output_per_gpu": 322.1475056289301, "total_per_gpu": 2908.374047034487, "tpot_ms": 47.08023753882907, "tpot_relative": 5.066969, "ttft_ms": 1674.3441834467376, "ttft_relative": 5.188765, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 89779.01194054476, "interactivity_tok_s": 11.502374744220221, "output_per_gpu": 366.19530165300944, "total_per_gpu": null, "tpot_error_pct": 12.6, "tpot_ms": 86.93856896833289, "tpot_relative": 9.356687, "ttft_error_pct": 76.74, "ttft_ms": 840.8558859402133, "ttft_relative": 2.605799, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 89812.44055380288, "interactivity_tok_s": 10.708834008968427, "output_per_gpu": 323.3727110407041, "total_per_gpu": 2909.517627915151, "tpot_error_pct": 20.95, "tpot_ms": 93.38084792074662, "tpot_relative": 10.050031, "ttft_error_pct": 1.49, "ttft_ms": 3668.956488245327, "ttft_relative": 11.370036, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 74836.25653850922, "interactivity_tok_s": 12.952172908050526, "output_per_gpu": 388.1984493926776, "total_per_gpu": 3492.7815275518187, "tpot_ms": 77.20712247274294, "tpot_relative": 8.309348, "ttft_ms": 3615.081174194711, "ttft_relative": 11.203077, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.17, "tpot_shape_error_pct": 5.25, "ttft_mape_pct": 37.54, "ttft_shape_error_pct": 9.32}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.16, "tpot_shape_error_pct": 4.88, "ttft_mape_pct": 55.74, "ttft_shape_error_pct": 3.48}, "framework": "trt", "id": "89ca0921f37b959a", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 17847.935836010838, "interactivity_tok_s": 58.58792106548978, "output_per_gpu": 114.52617185866774, "total_per_gpu": null, "tpot_error_pct": 21.09, "tpot_ms": 17.06836463581284, "tpot_relative": 0.78906, "ttft_error_pct": 34.48, "ttft_ms": 386.99881357430013, "ttft_relative": 0.655187, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16042.874415943912, "interactivity_tok_s": 57.942465218754954, "output_per_gpu": 111.12966089136954, "total_per_gpu": 1002.9173741151667, "tpot_error_pct": 20.21, "tpot_ms": 17.25849937907574, "tpot_relative": 0.79785, "ttft_error_pct": 56.5, "ttft_ms": 256.9462301125007, "ttft_relative": 0.435009, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 20384.336127531424, "interactivity_tok_s": 46.22940898475211, "output_per_gpu": 87.04913885128386, "total_per_gpu": 785.5966899877061, "tpot_ms": 21.63125209603763, "tpot_relative": 1.0, "ttft_ms": 590.6688590155682, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22621.5193957769, "interactivity_tok_s": 46.254257740658765, "output_per_gpu": 181.76602503510668, "total_per_gpu": null, "tpot_error_pct": 25.24, "tpot_ms": 21.619631334413835, "tpot_relative": 0.999463, "ttft_error_pct": 40.59, "ttft_ms": 504.63654067154766, "ttft_relative": 0.854348, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20688.65600790943, "interactivity_tok_s": 45.56724859334344, "output_per_gpu": 174.72777191397643, "total_per_gpu": 1562.4713709979703, "tpot_error_pct": 24.11, "tpot_ms": 21.945586597170188, "tpot_relative": 1.014531, "ttft_error_pct": 54.98, "ttft_ms": 382.3701782843758, "ttft_relative": 0.647351, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 27622.96944420741, "interactivity_tok_s": 34.58054094143818, "output_per_gpu": 130.63896556125184, "total_per_gpu": 1168.2152264079682, "tpot_ms": 28.917997601410878, "tpot_relative": 1.336862, "ttft_ms": 849.4033809991379, "ttft_relative": 1.438037, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 66.67, "points": 2, "status_counts": {"failed": 1, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.7, "tpot_shape_error_pct": 4.94, "ttft_mape_pct": 7.38, "ttft_shape_error_pct": 10.9}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.13, "tpot_shape_error_pct": 20.3, "ttft_mape_pct": 305.93, "ttft_shape_error_pct": 704.38}, "framework": "trt", "id": "b7049622b51011ac", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 20789.639159432765, "interactivity_tok_s": 50.51382514213324, "output_per_gpu": 97.75769553340498, "total_per_gpu": null, "tpot_error_pct": 36.06, "tpot_ms": 19.796560588833863, "tpot_relative": 1.360566, "ttft_error_pct": 13.56, "ttft_ms": 537.7576770557248, "ttft_relative": 1.135632, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18886.870091387464, "interactivity_tok_s": 49.8612366987834, "output_per_gpu": 96.59791502616714, "total_per_gpu": 859.2831151576362, "tpot_error_pct": 37.84, "tpot_ms": 20.05565979121412, "tpot_relative": 1.378373, "ttft_error_pct": 32.54, "ttft_ms": 319.44164381250187, "ttft_relative": 0.674594, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 13955.485713010421, "interactivity_tok_s": 68.72739392520563, "output_per_gpu": 130.70262718593216, "total_per_gpu": 1162.660297763067, "tpot_ms": 14.550238891471366, "tpot_relative": 1.0, "ttft_ms": 473.5316864476772, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27194.445262636887, "interactivity_tok_s": 38.515875906583375, "output_per_gpu": 150.29612742123692, "total_per_gpu": null, "tpot_error_pct": 29.33, "tpot_ms": 25.963319708096623, "tpot_relative": 1.784391, "ttft_error_pct": 1.19, "ttft_ms": 633.9692012540421, "ttft_relative": 1.338811, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26719.298359056214, "interactivity_tok_s": 41.120845904687485, "output_per_gpu": 133.26236949571305, "total_per_gpu": 1202.6595295170664, "tpot_error_pct": 21.14, "tpot_ms": 24.31856587575712, "tpot_relative": 1.671352, "ttft_error_pct": 614.03, "ttft_ms": 4473.657709199995, "ttft_relative": 9.447431, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 18983.90301173349, "interactivity_tok_s": 49.814126279388404, "output_per_gpu": 188.00266832944973, "total_per_gpu": 1696.677025154672, "tpot_ms": 20.074626911880017, "tpot_relative": 1.379677, "ttft_ms": 626.5326763968915, "ttft_relative": 1.323106, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 53286.6883718371, "interactivity_tok_s": 41.08373955137123, "output_per_gpu": 133.5588683437196, "total_per_gpu": 1194.3259268062388, "tpot_error_pct": 1.42, "tpot_ms": 24.34053012018531, "tpot_relative": 1.672861, "ttft_error_pct": 271.22, "ttft_ms": 30782.985040059102, "ttft_relative": 65.007234, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016874169", "measured": {"e2e_ms": 31143.667129242385, "interactivity_tok_s": 40.49894275187963, "output_per_gpu": 230.35313253156943, "total_per_gpu": 2059.891057143896, "tpot_ms": 24.692002606749238, "tpot_relative": 1.697017, "ttft_ms": 8292.347574694577, "ttft_relative": 17.511706, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.64, "tpot_shape_error_pct": 3.39, "ttft_mape_pct": 17.5, "ttft_shape_error_pct": 28.39}, "aisimulate": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.39, "tpot_shape_error_pct": 2.92, "ttft_mape_pct": 49.27, "ttft_shape_error_pct": 19.24}, "gpu": "b300", "precisions": ["fp8"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.14, "tpot_shape_error_pct": 3.01, "ttft_mape_pct": 20.22, "ttft_shape_error_pct": 31.55}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.8, "tpot_shape_error_pct": 3.04, "ttft_mape_pct": 51.4, "ttft_shape_error_pct": 20.23}, "framework": "sglang", "id": "b10022a93289f208", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7646.015128648848, "interactivity_tok_s": 142.24559626604128, "output_per_gpu": 34.41730767845867, "total_per_gpu": null, "tpot_error_pct": 9.4, "tpot_ms": 7.030094612768923, "tpot_relative": 0.906024, "ttft_error_pct": 19.25, "ttft_ms": 454.2283397862404, "ttft_relative": 1.192531, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6742.103199600005, "interactivity_tok_s": 142.54269169446533, "output_per_gpu": 34.495912197516965, "total_per_gpu": 308.790882958231, "tpot_error_pct": 9.59, "tpot_ms": 7.015442097469723, "tpot_relative": 0.904135, "ttft_error_pct": 41.55, "ttft_ms": 222.6442313000004, "ttft_relative": 0.58453, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7591.725219879299, "interactivity_tok_s": 128.87788427875043, "output_per_gpu": 30.63397704156401, "total_per_gpu": 274.2206892522073, "tpot_ms": 7.759283181877012, "tpot_relative": 1.0, "ttft_ms": 380.8942117029801, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9106.025971917923, "interactivity_tok_s": 118.40564590935078, "output_per_gpu": 57.60858020309816, "total_per_gpu": null, "tpot_error_pct": 4.81, "tpot_ms": 8.445543219835834, "tpot_relative": 1.088444, "ttft_error_pct": 5.09, "ttft_ms": 466.2352580258646, "ttft_relative": 1.224054, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8102.8599025, "interactivity_tok_s": 118.74851092868398, "output_per_gpu": 57.55852529782877, "total_per_gpu": 512.9309288088629, "tpot_error_pct": 5.09, "tpot_ms": 8.42115822909614, "tpot_relative": 1.085301, "ttft_error_pct": 47.31, "ttft_ms": 233.76787485000014, "ttft_relative": 0.613734, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8737.338940799236, "interactivity_tok_s": 112.70921889636364, "output_per_gpu": 53.38780404691307, "total_per_gpu": 475.76368183779164, "tpot_ms": 8.87238869891825, "tpot_relative": 1.143455, "ttft_ms": 443.66020465968177, "ttft_relative": 1.164786, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10772.143055241168, "interactivity_tok_s": 99.49536765876026, "output_per_gpu": 93.17589881507037, "total_per_gpu": null, "tpot_error_pct": 9.55, "tpot_ms": 10.050719179506977, "tpot_relative": 1.295315, "ttft_error_pct": 6.75, "ttft_ms": 490.2573346055302, "ttft_relative": 1.287122, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9843.587497149976, "interactivity_tok_s": 95.87059994048194, "output_per_gpu": 91.51361861367253, "total_per_gpu": 822.6528538629705, "tpot_error_pct": 6.13, "tpot_ms": 10.430726423124677, "tpot_relative": 1.34429, "ttft_error_pct": 51.27, "ttft_ms": 256.223469949999, "ttft_relative": 0.672689, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 10738.50804787503, "interactivity_tok_s": 89.99071605990629, "output_per_gpu": 83.82075759164971, "total_per_gpu": 753.4986211923724, "tpot_ms": 11.112257394799547, "tpot_relative": 1.432124, "ttft_ms": 525.7726444251148, "ttft_relative": 1.380364, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13946.756241886886, "interactivity_tok_s": 76.29533258251402, "output_per_gpu": 145.1065153725839, "total_per_gpu": null, "tpot_error_pct": 11.34, "tpot_ms": 13.106961673159912, "tpot_relative": 1.689197, "ttft_error_pct": 18.15, "ttft_ms": 538.3344502442966, "ttft_relative": 1.413344, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12761.129876225052, "interactivity_tok_s": 74.35911239726877, "output_per_gpu": 143.04475734406262, "total_per_gpu": 1272.4492517698536, "tpot_error_pct": 9.03, "tpot_ms": 13.448250897044465, "tpot_relative": 1.733182, "ttft_error_pct": 53.37, "ttft_ms": 306.69366358749994, "ttft_relative": 0.805194, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 14353.25977296452, "interactivity_tok_s": 67.64694981886308, "output_per_gpu": 127.17709319612662, "total_per_gpu": 1131.2990429312872, "tpot_ms": 14.782632515991933, "tpot_relative": 1.905154, "ttft_ms": 657.6699204742908, "ttft_relative": 1.726647, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20195.603835160713, "interactivity_tok_s": 52.297984461481335, "output_per_gpu": 202.16213438407348, "total_per_gpu": null, "tpot_error_pct": 10.39, "tpot_ms": 19.121195784065502, "tpot_relative": 2.464299, "ttft_error_pct": 31.48, "ttft_ms": 634.6205480617011, "ttft_relative": 1.666133, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18194.020751206197, "interactivity_tok_s": 51.4240659893099, "output_per_gpu": 196.64624645317028, "total_per_gpu": 1774.6831542588986, "tpot_error_pct": 8.87, "tpot_ms": 19.44614803908896, "tpot_relative": 2.506178, "ttft_error_pct": 56.67, "ttft_ms": 401.3058598375, "ttft_relative": 1.053589, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 20461.567144798755, "interactivity_tok_s": 46.862568722870066, "output_per_gpu": 174.90422741507933, "total_per_gpu": 1578.4668744039714, "tpot_ms": 21.33899244648055, "tpot_relative": 2.750124, "ttft_ms": 926.1219516360143, "ttft_relative": 2.431441, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28783.2470125972, "interactivity_tok_s": 36.593837493551696, "output_per_gpu": 285.9475891729788, "total_per_gpu": null, "tpot_error_pct": 15.38, "tpot_ms": 27.327005542291452, "tpot_relative": 3.521847, "ttft_error_pct": 40.59, "ttft_ms": 827.7203428330447, "ttft_relative": 2.173098, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26257.94814410955, "interactivity_tok_s": 36.04721643584079, "output_per_gpu": 276.4591036935437, "total_per_gpu": 2472.1853317375753, "tpot_error_pct": 14.1, "tpot_ms": 27.741393063729788, "tpot_relative": 3.575252, "ttft_error_pct": 58.25, "ttft_ms": 581.7144381374995, "ttft_relative": 1.527234, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 31298.04432281817, "interactivity_tok_s": 30.96614129207452, "output_per_gpu": 231.53123017634564, "total_per_gpu": 2070.425981398005, "tpot_ms": 32.29333582663527, "tpot_relative": 4.161897, "ttft_ms": 1393.2973334944109, "ttft_relative": 3.657964, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.62, "tpot_shape_error_pct": 4.34, "ttft_mape_pct": 12.05, "ttft_shape_error_pct": 20.46}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.58, "tpot_shape_error_pct": 2.61, "ttft_mape_pct": 45.0, "ttft_shape_error_pct": 16.75}, "framework": "sglang", "id": "e15127d62a71a9b8", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7314.509010662051, "interactivity_tok_s": 147.07181174115357, "output_per_gpu": 17.898164354089932, "total_per_gpu": null, "tpot_error_pct": 5.9, "tpot_ms": 6.799399478127055, "tpot_relative": 0.941024, "ttft_error_pct": 24.53, "ttft_ms": 358.7233445380743, "ttft_relative": 1.245295, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6479.8899186, "interactivity_tok_s": 147.47410876346189, "output_per_gpu": 17.945906714588798, "total_per_gpu": 160.64316108396181, "tpot_error_pct": 6.15, "tpot_ms": 6.780851285590272, "tpot_relative": 0.938457, "ttft_error_pct": 38.08, "ttft_ms": 178.36811130000018, "ttft_relative": 0.619198, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7002.845927583985, "interactivity_tok_s": 138.39804040382978, "output_per_gpu": 16.605054846847555, "total_per_gpu": 148.64049741224088, "tpot_ms": 7.225535831881097, "tpot_relative": 1.0, "ttft_ms": 288.06301343720406, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7758.8286183167775, "interactivity_tok_s": 138.41846503466536, "output_per_gpu": 33.742549426983594, "total_per_gpu": null, "tpot_error_pct": 10.57, "tpot_ms": 7.224469652582559, "tpot_relative": 0.999852, "ttft_error_pct": 4.86, "ttft_ms": 368.19616372481994, "ttft_relative": 1.278179, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6925.300899499996, "interactivity_tok_s": 138.6933953469026, "output_per_gpu": 33.66116974553376, "total_per_gpu": 299.9704209416347, "tpot_error_pct": 10.75, "tpot_ms": 7.210148670012589, "tpot_relative": 0.99787, "ttft_error_pct": 46.53, "ttft_ms": 187.7583732500001, "ttft_relative": 0.651796, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7900.536439067218, "interactivity_tok_s": 123.78636623637183, "output_per_gpu": 29.518164540295878, "total_per_gpu": 263.05016461146107, "tpot_ms": 8.07843408288184, "tpot_relative": 1.118039, "ttft_ms": 351.1308564338833, "ttft_relative": 1.218938, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8874.97986079296, "interactivity_tok_s": 120.52546242401286, "output_per_gpu": 56.59566069058516, "total_per_gpu": null, "tpot_error_pct": 9.39, "tpot_ms": 8.29700197690978, "tpot_relative": 1.148289, "ttft_error_pct": 6.77, "ttft_ms": 387.14683841425546, "ttft_relative": 1.343966, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8131.114618725007, "interactivity_tok_s": 115.98060274725522, "output_per_gpu": 55.30521418353801, "total_per_gpu": 497.16089223460045, "tpot_error_pct": 5.84, "tpot_ms": 8.622131428125087, "tpot_relative": 1.193286, "ttft_error_pct": 50.38, "ttft_ms": 206.0444886999998, "ttft_relative": 0.715276, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8831.473643002391, "interactivity_tok_s": 109.20493412525447, "output_per_gpu": 50.86462648368232, "total_per_gpu": 457.2426571188348, "tpot_ms": 9.15709540058907, "tpot_relative": 1.267324, "ttft_ms": 415.2550363985938, "ttft_relative": 1.441542, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.23, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.23, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "05b632fe6024ea55", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 30978.99, "interactivity_tok_s": 34.21610894409088, "output_per_gpu": 288.0914285714286, "total_per_gpu": null, "tpot_error_pct": 74.28, "tpot_ms": 29.226, "tpot_relative": 1.742795, "ttft_error_pct": 41.23, "ttft_ms": 1080.792, "ttft_relative": 0.587724, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 17286.303497733345, "interactivity_tok_s": 59.631651161032806, "output_per_gpu": 811.3311097528446, "total_per_gpu": 4172.8593206659625, "tpot_ms": 16.769617820904227, "tpot_relative": 1.0, "ttft_ms": 1838.9446501254497, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "51ec1ea603869da9", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 28375.668111002276, "interactivity_tok_s": 41.36038314058601, "output_per_gpu": 2378.6800938763377, "total_per_gpu": 7781.744649200224, "tpot_ms": 24.177725738200973, "tpot_relative": 1.0, "ttft_ms": 6105.478990258648, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "98f29719e4816e05", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 7557.915965403663, "interactivity_tok_s": 135.341468893667, "output_per_gpu": 14.807663201774206, "total_per_gpu": 118.33613706232126, "tpot_ms": 7.388718388934174, "tpot_relative": 1.0, "ttft_ms": 765.9407191269565, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 13529.183720804092, "interactivity_tok_s": 103.0254359330046, "output_per_gpu": 48.69777546165318, "total_per_gpu": 389.53804058564964, "tpot_ms": 9.706340875376448, "tpot_relative": 1.31367, "ttft_ms": 4618.0365122214425, "ttft_relative": 6.029235, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 36, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 12519.33512077473, "interactivity_tok_s": 87.55353002142907, "output_per_gpu": 78.8791287727118, "total_per_gpu": 626.0507807452734, "tpot_ms": 11.421584026997497, "tpot_relative": 1.545814, "ttft_ms": 1943.8799579175086, "ttft_relative": 2.537899, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "6829910241c57dcc", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 50396.36938828887, "interactivity_tok_s": 20.73594842943921, "output_per_gpu": 4741.775421167531, "total_per_gpu": 9476.255773647665, "tpot_ms": 48.22542857891571, "tpot_relative": 1.0, "ttft_ms": 5959.701934609979, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "63c1d97c6cf64d1c", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27595478969", "measured": {"e2e_ms": 27364.13042236601, "interactivity_tok_s": 46.03493608875994, "output_per_gpu": 1335.77814030689, "total_per_gpu": 6014.782580263403, "tpot_ms": 21.722632525695275, "tpot_relative": 1.0, "ttft_ms": 7365.619208987952, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}]}, {"aic": {"coverage_pct": 89.47, "points": 17, "status_counts": {"failed": 2, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.3, "tpot_shape_error_pct": 10.96, "ttft_mape_pct": 51.76, "ttft_shape_error_pct": 29.95}, "aisimulate": {"coverage_pct": 52.63, "points": 10, "status_counts": {"failed": 9, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.4, "tpot_shape_error_pct": 7.43, "ttft_mape_pct": 21.29, "ttft_shape_error_pct": 9.49}, "gpu": "h200", "precisions": ["fp8"], "rows": 19, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.19, "tpot_shape_error_pct": null, "ttft_mape_pct": 92.85, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "8549ea31fc3192eb", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 48902.407, "interactivity_tok_s": 21.190930281839375, "output_per_gpu": 206.712, "total_per_gpu": null, "tpot_error_pct": 7.19, "tpot_ms": 47.19, "tpot_relative": 1.071876, "ttft_error_pct": 92.85, "ttft_ms": 627.037, "ttft_relative": 0.071547, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 768, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 49301.23521190053, "interactivity_tok_s": 22.714046344518827, "output_per_gpu": 573.7573294242637, "total_per_gpu": 1937.276734576658, "tpot_ms": 44.0256211875394, "tpot_relative": 1.0, "ttft_ms": 8763.985807768737, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 73.35, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "8aa1a12e1e2a2d03", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 29213.749, "interactivity_tok_s": 35.785857429144, "output_per_gpu": 94.12389285714286, "total_per_gpu": null, "tpot_error_pct": 1.42, "tpot_ms": 27.944, "tpot_relative": 1.014196, "ttft_error_pct": 73.35, "ttft_ms": 627.037, "ttft_relative": 0.266472, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 160, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 27675.512999614584, "interactivity_tok_s": 36.29386273246281, "output_per_gpu": 128.5027367312207, "total_per_gpu": 828.399830050411, "tpot_ms": 27.552867749884236, "tpot_relative": 1.0, "ttft_ms": 2353.108334388089, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 50.0, "points": 1, "status_counts": {"failed": 1, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.27, "tpot_shape_error_pct": null, "ttft_mape_pct": 42.72, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "840d65537dfed246", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 9, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 13062.548941844661, "interactivity_tok_s": 91.35329135321628, "output_per_gpu": 10.930061952668547, "total_per_gpu": 85.53140081237672, "tpot_ms": 10.946513094241054, "tpot_relative": 1.0, "ttft_ms": 3011.131090246555, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 13010.452, "interactivity_tok_s": 82.61049153242462, "output_per_gpu": 33.281921875, "total_per_gpu": null, "tpot_error_pct": 12.27, "tpot_ms": 12.105, "tpot_relative": 1.105832, "ttft_error_pct": 42.72, "ttft_ms": 627.037, "ttft_relative": 0.20824, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 28, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 13792.76966869199, "interactivity_tok_s": 72.47743985229243, "output_per_gpu": 31.976956909137566, "total_per_gpu": 251.57969397189174, "tpot_ms": 13.797396845666457, "tpot_relative": 1.260438, "ttft_ms": 1094.759165272782, "ttft_relative": 0.363571, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "84aecae6b4ca2334", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 57642.62544073104, "interactivity_tok_s": 17.160095393637242, "output_per_gpu": 983.4343915057989, "total_per_gpu": 2212.885699031083, "tpot_ms": 58.27473432174439, "tpot_relative": 1.0, "ttft_ms": 3970.0096148385455, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.57, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.85, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "fdba2047c820b5c6", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 39128.665, "interactivity_tok_s": 26.57030502710171, "output_per_gpu": 165.369625, "total_per_gpu": null, "tpot_error_pct": 3.57, "tpot_ms": 37.636, "tpot_relative": 1.035722, "ttft_error_pct": 91.85, "ttft_ms": 627.037, "ttft_relative": 0.081531, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 41086.16411576131, "interactivity_tok_s": 27.51944825740749, "output_per_gpu": 343.41707648968026, "total_per_gpu": 1550.3853549515768, "tpot_ms": 36.33793783386726, "tpot_relative": 1.0, "ttft_ms": 7690.781913593855, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.2, "tpot_shape_error_pct": null, "ttft_mape_pct": 63.51, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "01ab84c3c2573149", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 17613.952, "interactivity_tok_s": 60.222824450466724, "output_per_gpu": 47.24846428571429, "total_per_gpu": null, "tpot_error_pct": 1.2, "tpot_ms": 16.605, "tpot_relative": 0.987991, "ttft_error_pct": 63.51, "ttft_ms": 627.037, "ttft_relative": 0.364874, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 48, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 17267.86222997956, "interactivity_tok_s": 59.49963714178738, "output_per_gpu": 51.82197354510995, "total_per_gpu": 398.2161630250575, "tpot_ms": 16.80682518478229, "tpot_relative": 1.0, "ttft_ms": 1718.5034038237063, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.41, "tpot_shape_error_pct": null, "ttft_mape_pct": 75.88, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "4f175c648d2bfebd", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 33678.121, "interactivity_tok_s": 30.9520861706079, "output_per_gpu": 132.29567500000002, "total_per_gpu": null, "tpot_error_pct": 2.41, "tpot_ms": 32.308, "tpot_relative": 0.975872, "ttft_error_pct": 75.88, "ttft_ms": 627.037, "ttft_relative": 0.241229, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 33056.17923729745, "interactivity_tok_s": 30.205279386557045, "output_per_gpu": 214.2808123344959, "total_per_gpu": 1158.7647909685006, "tpot_ms": 33.106795246034146, "tpot_relative": 1.0, "ttft_ms": 2599.341617976218, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.06, "tpot_shape_error_pct": null, "ttft_mape_pct": 56.72, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "be5e15fb9c372711", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 21594.445, "interactivity_tok_s": 48.79000780640125, "output_per_gpu": 67.38815625, "total_per_gpu": null, "tpot_error_pct": 4.06, "tpot_ms": 20.496, "tpot_relative": 0.959409, "ttft_error_pct": 56.72, "ttft_ms": 627.037, "ttft_relative": 0.432811, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 48, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 21215.439242718276, "interactivity_tok_s": 46.80958479554539, "output_per_gpu": 84.63222374861289, "total_per_gpu": 569.0477893050465, "tpot_ms": 21.363146124192166, "tpot_relative": 1.0, "ttft_ms": 1448.7552610303587, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.52, "tpot_shape_error_pct": 13.0, "ttft_mape_pct": 54.18, "ttft_shape_error_pct": 37.27}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.52, "tpot_shape_error_pct": 7.88, "ttft_mape_pct": 12.58, "ttft_shape_error_pct": 14.6}, "framework": "sglang", "id": "a691138166eafdb9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 11948.234669926098, "interactivity_tok_s": 91.32446172180305, "output_per_gpu": 41.67944912424722, "total_per_gpu": null, "tpot_error_pct": 0.11, "tpot_ms": 10.949968728490816, "tpot_relative": 1.001068, "ttft_error_pct": 99.97, "ttft_ms": 746.4166606799934, "ttft_relative": 1.999698, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10913.31063544999, "interactivity_tok_s": 87.15128429622028, "output_per_gpu": 41.27826373304848, "total_per_gpu": 371.0669731666283, "tpot_error_pct": 4.9, "tpot_ms": 11.474300213420603, "tpot_relative": 1.049004, "ttft_error_pct": 1.87, "ttft_ms": 366.2738712750007, "ttft_relative": 0.981271, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 10424.170383880846, "interactivity_tok_s": 91.42202375222126, "output_per_gpu": 43.11504596173044, "total_per_gpu": 387.5785499706106, "tpot_ms": 10.938283347459842, "tpot_relative": 1.0, "ttft_ms": 373.26465928927064, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15940.873120121585, "interactivity_tok_s": 67.65353422825879, "output_per_gpu": 63.20843798115196, "total_per_gpu": null, "tpot_error_pct": 6.52, "tpot_ms": 14.781193789907006, "tpot_relative": 1.351327, "ttft_error_pct": 67.11, "ttft_ms": 819.7118730467192, "ttft_relative": 2.196061, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14430.064733162519, "interactivity_tok_s": 66.20662091282294, "output_per_gpu": 63.288036395104875, "total_per_gpu": 562.9763442726904, "tpot_error_pct": 8.85, "tpot_ms": 15.104229550043678, "tpot_relative": 1.380859, "ttft_error_pct": 10.85, "ttft_ms": 437.2830972124995, "ttft_relative": 1.17151, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 13341.410665772855, "interactivity_tok_s": 72.06504164699513, "output_per_gpu": 68.33426952403, "total_per_gpu": 607.8649210257171, "tpot_ms": 13.876353598717397, "tpot_relative": 1.268604, "ttft_ms": 490.5263568041846, "ttft_relative": 1.314152, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25494.520982534825, "interactivity_tok_s": 41.70749515603915, "output_per_gpu": 80.00242040744213, "total_per_gpu": null, "tpot_error_pct": 19.79, "tpot_ms": 23.976505811694672, "tpot_relative": 2.191981, "ttft_error_pct": 40.45, "ttft_ms": 966.5555371711753, "ttft_relative": 2.589464, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22645.022897531293, "interactivity_tok_s": 41.46991266625769, "output_per_gpu": 79.20634252609129, "total_per_gpu": 714.8174161818498, "tpot_error_pct": 20.48, "tpot_ms": 24.11386799986337, "tpot_relative": 2.204539, "ttft_error_pct": 15.94, "ttft_ms": 578.4986365437503, "ttft_relative": 1.549835, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 19008.522158995038, "interactivity_tok_s": 49.96127819157269, "output_per_gpu": 93.88911672015439, "total_per_gpu": 847.3257782277859, "tpot_ms": 20.01550072769509, "tpot_relative": 1.829858, "ttft_ms": 688.2046255806927, "ttft_relative": 1.843744, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37802.988334706846, "interactivity_tok_s": 27.99538919803659, "output_per_gpu": 108.92528150723246, "total_per_gpu": null, "tpot_error_pct": 13.86, "tpot_ms": 35.72016780785221, "tpot_relative": 3.26561, "ttft_error_pct": 28.79, "ttft_ms": 1261.256667274039, "ttft_relative": 3.378988, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33735.6894112125, "interactivity_tok_s": 28.139784543089704, "output_per_gpu": 107.83286110288938, "total_per_gpu": 964.2757787183124, "tpot_error_pct": 13.27, "tpot_ms": 35.53687479265261, "tpot_relative": 3.248853, "ttft_error_pct": 14.28, "ttft_ms": 839.5137575343748, "ttft_relative": 2.249111, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 30021.117252943804, "interactivity_tok_s": 31.8748044980678, "output_per_gpu": 121.01106790110163, "total_per_gpu": 1082.1194999410313, "tpot_ms": 31.372741440990435, "tpot_relative": 2.86816, "ttft_ms": 979.3433630402433, "ttft_relative": 2.623724, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 58144.69012989958, "interactivity_tok_s": 18.050094269822488, "output_per_gpu": 141.91920003025754, "total_per_gpu": null, "tpot_error_pct": 12.3, "tpot_ms": 55.40137270484374, "tpot_relative": 5.064906, "ttft_error_pct": 34.58, "ttft_ms": 1469.0858528444367, "ttft_relative": 3.935775, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52818.193438020404, "interactivity_tok_s": 18.415270062271127, "output_per_gpu": 136.2892493001883, "total_per_gpu": 1228.4810336965786, "tpot_error_pct": 10.08, "tpot_ms": 54.302760514426666, "tpot_relative": 4.964468, "ttft_error_pct": 19.97, "ttft_ms": 2694.029097653129, "ttft_relative": 7.217477, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25980019087", "measured": {"e2e_ms": 47755.7807585923, "interactivity_tok_s": 20.27103829948501, "output_per_gpu": 150.9772623797434, "total_per_gpu": 1360.8755225030834, "tpot_ms": 49.331464191718545, "tpot_relative": 4.509982, "ttft_ms": 2245.575932835345, "ttft_relative": 6.016042, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 11.72, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 183.9, "tpot_shape_error_pct": null, "ttft_mape_pct": 50.41, "ttft_shape_error_pct": null}, "framework": "trt", "id": "441971b3b48048ae", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 43284.89369887693, "interactivity_tok_s": 26.084814171030608, "output_per_gpu": 183.64690552862092, "total_per_gpu": null, "tpot_error_pct": 11.5, "tpot_ms": 38.336481657231225, "tpot_relative": 0.885019, "ttft_error_pct": 11.72, "ttft_ms": 4066.672963529384, "ttft_relative": 0.882828, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 115855.5398249845, "interactivity_tok_s": 8.131685392420689, "output_per_gpu": 63.170401178284784, "total_per_gpu": 569.4039708707945, "tpot_error_pct": 183.9, "tpot_ms": 122.97573648533812, "tpot_relative": 2.838965, "ttft_error_pct": 50.41, "ttft_ms": 2284.462810518751, "ttft_relative": 0.495931, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 44510.56280010089, "interactivity_tok_s": 23.08556784359364, "output_per_gpu": 162.38093847817896, "total_per_gpu": 1463.6657269636764, "tpot_ms": 43.31710646127792, "tpot_relative": 1.0, "ttft_ms": 4606.413363380125, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.99, "tpot_shape_error_pct": 8.24, "ttft_mape_pct": 25.11, "ttft_shape_error_pct": 20.2}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.62, "tpot_shape_error_pct": 6.82, "ttft_mape_pct": 24.89, "ttft_shape_error_pct": 2.68}, "framework": "trt", "id": "53ec2bd0a2e7c8bb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12058.712510744257, "interactivity_tok_s": 89.5701060482867, "output_per_gpu": 41.478108172338985, "total_per_gpu": null, "tpot_error_pct": 12.4, "tpot_ms": 11.164439165237853, "tpot_relative": 0.875993, "ttft_error_pct": 47.45, "ttft_ms": 637.4912447059316, "ttft_relative": 1.47446, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11006.506215775018, "interactivity_tok_s": 86.09378166888351, "output_per_gpu": 41.003142048603, "total_per_gpu": 368.5937933022836, "tpot_error_pct": 8.86, "tpot_ms": 11.615240736502873, "tpot_relative": 0.911364, "ttft_error_pct": 23.62, "ttft_ms": 330.21571112500015, "ttft_relative": 0.763759, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 12141.689230431803, "interactivity_tok_s": 78.46279260908982, "output_per_gpu": 37.071696314648385, "total_per_gpu": 333.2524408146445, "tpot_ms": 12.744894321848943, "tpot_relative": 1.0, "ttft_ms": 432.3558239033446, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17411.50686864374, "interactivity_tok_s": 61.21532783559507, "output_per_gpu": 58.084833448007956, "total_per_gpu": null, "tpot_error_pct": 1.15, "tpot_ms": 16.335777906567493, "tpot_relative": 1.281751, "ttft_error_pct": 31.76, "ttft_ms": 700.0060702251938, "ttft_relative": 1.619051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15851.933790112535, "interactivity_tok_s": 59.951786611067114, "output_per_gpu": 57.67444285859868, "total_per_gpu": 513.0408343812901, "tpot_error_pct": 0.93, "tpot_ms": 16.680070045075183, "tpot_relative": 1.308765, "ttft_error_pct": 24.29, "ttft_ms": 402.2517814624997, "ttft_relative": 0.930372, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 15839.491749013541, "interactivity_tok_s": 60.50835834331831, "output_per_gpu": 57.59124901185511, "total_per_gpu": 512.300786650732, "tpot_ms": 16.52664239089253, "tpot_relative": 1.296727, "ttft_ms": 531.2731346522924, "ttft_relative": 1.228787, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23208.44024640038, "interactivity_tok_s": 45.703852465542006, "output_per_gpu": 87.91050206477182, "total_per_gpu": null, "tpot_error_pct": 12.03, "tpot_ms": 21.87999361222209, "tpot_relative": 1.716765, "ttft_error_pct": 13.77, "ttft_ms": 825.2067810971836, "ttft_relative": 1.908629, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20821.190046599986, "interactivity_tok_s": 45.066848084382976, "output_per_gpu": 85.9936320792823, "total_per_gpu": 776.0710055606535, "tpot_error_pct": 10.78, "tpot_ms": 22.189259788650055, "tpot_relative": 1.741031, "ttft_error_pct": 28.55, "ttft_ms": 518.2038879187501, "ttft_relative": 1.198559, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 23484.099143295316, "interactivity_tok_s": 40.20709039415855, "output_per_gpu": 76.16118381602456, "total_per_gpu": 687.335621017826, "tpot_ms": 24.871235152725305, "tpot_relative": 1.951467, "ttft_ms": 725.3111328172963, "ttft_relative": 1.677579, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34377.684393188356, "interactivity_tok_s": 30.719436824671337, "output_per_gpu": 119.74648497694487, "total_per_gpu": null, "tpot_error_pct": 2.37, "tpot_ms": 32.55268010632545, "tpot_relative": 2.554174, "ttft_error_pct": 7.45, "ttft_ms": 1076.2926444174252, "ttft_relative": 2.489368, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31040.430228159425, "interactivity_tok_s": 30.58019865805976, "output_per_gpu": 117.10992114742886, "total_per_gpu": 1047.2342035173106, "tpot_error_pct": 1.92, "tpot_ms": 32.700899401660315, "tpot_relative": 2.565804, "ttft_error_pct": 23.08, "ttft_ms": 770.4881476562516, "ttft_relative": 1.78207, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016868638", "measured": {"e2e_ms": 31867.946812536684, "interactivity_tok_s": 29.991990712410118, "output_per_gpu": 113.77933411142469, "total_per_gpu": 1017.451033759182, "tpot_ms": 33.34223491827833, "tpot_relative": 2.616125, "ttft_ms": 1001.6335342166714, "ttft_relative": 2.316688, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 64}]}, {"aic": {"coverage_pct": 66.0, "points": 66, "status_counts": {"failed": 34, "success": 66, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.97, "tpot_shape_error_pct": 16.26, "ttft_mape_pct": 48.45, "ttft_shape_error_pct": 45.41}, "aisimulate": {"coverage_pct": 60.0, "points": 60, "status_counts": {"failed": 40, "success": 60, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.59, "tpot_shape_error_pct": 14.86, "ttft_mape_pct": 15.74, "ttft_shape_error_pct": 29.44}, "frameworks": ["dynamo-sglang", "dynamo-vllm", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300"], "hf_model_paths": ["deepseek-ai/DeepSeek-V4-Pro"], "model": "deepseek-ai/DeepSeek-V4-Pro", "precisions": ["fp4"], "rows": 100, "workloads": [{"aic": {"coverage_pct": 85.71, "points": 18, "status_counts": {"failed": 3, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.15, "tpot_shape_error_pct": 13.59, "ttft_mape_pct": 64.64, "ttft_shape_error_pct": 32.13}, "aisimulate": {"coverage_pct": 85.71, "points": 18, "status_counts": {"failed": 3, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.2, "tpot_shape_error_pct": 12.52, "ttft_mape_pct": 16.37, "ttft_shape_error_pct": 12.79}, "gpu_skus": ["b300", "gb200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.15, "tpot_shape_error_pct": 13.59, "ttft_mape_pct": 64.64, "ttft_shape_error_pct": 32.13}, "aisimulate": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.2, "tpot_shape_error_pct": 12.52, "ttft_mape_pct": 16.37, "ttft_shape_error_pct": 12.79}, "gpu": "b300", "precisions": ["fp4"], "rows": 18, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.18, "tpot_shape_error_pct": 3.94, "ttft_mape_pct": 65.04, "ttft_shape_error_pct": 29.7}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.6, "tpot_shape_error_pct": 1.33, "ttft_mape_pct": 34.72, "ttft_shape_error_pct": 11.02}, "framework": "vllm", "id": "09cf5f359daa19ac", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 48010.616034862964, "interactivity_tok_s": 21.440849206978818, "output_per_gpu": 1363.698394381306, "total_per_gpu": null, "tpot_error_pct": 36.94, "tpot_ms": 46.639943704958675, "tpot_relative": 0.630618, "ttft_error_pct": 58.94, "ttft_ms": 297.9536246902414, "ttft_relative": 0.410572, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56974.3823827412, "interactivity_tok_s": 16.319789719768554, "output_per_gpu": 1011.6973721577629, "total_per_gpu": 2022.416942086824, "tpot_error_pct": 17.15, "tpot_ms": 61.275299325007595, "tpot_relative": 0.828502, "ttft_error_pct": 38.13, "ttft_ms": 449.0235567859372, "ttft_relative": 0.618742, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24920699270", "measured": {"e2e_ms": 68930.81857004481, "interactivity_tok_s": 13.520981775631858, "output_per_gpu": 834.6907361831054, "total_per_gpu": 1668.574746477049, "tpot_ms": 73.95912638550008, "tpot_relative": 1.0, "ttft_ms": 725.7042055691272, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 61138.517813819984, "interactivity_tok_s": 16.822494907198255, "output_per_gpu": 2141.7594780225586, "total_per_gpu": null, "tpot_error_pct": 39.42, "tpot_ms": 59.44421475628477, "tpot_relative": 0.803744, "ttft_error_pct": 71.14, "ttft_ms": 327.08611814066325, "ttft_relative": 0.450715, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 76788.16780347975, "interactivity_tok_s": 12.13816388590108, "output_per_gpu": 1506.6344562687923, "total_per_gpu": 3011.881032440628, "tpot_error_pct": 16.05, "tpot_ms": 82.38478318467396, "tpot_relative": 1.113923, "ttft_error_pct": 31.31, "ttft_ms": 778.3832823755846, "ttft_relative": 1.07259, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24920699270", "measured": {"e2e_ms": 91653.5653144023, "interactivity_tok_s": 10.190500745961756, "output_per_gpu": 1259.8225777171338, "total_per_gpu": 2518.484633269097, "tpot_ms": 98.13060466103937, "tpot_relative": 1.326822, "ttft_ms": 1133.1875266153702, "ttft_relative": 1.561501, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.72, "tpot_shape_error_pct": 17.33, "ttft_mape_pct": 54.99, "ttft_shape_error_pct": 26.38}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.58, "tpot_shape_error_pct": 18.09, "ttft_mape_pct": 6.36, "ttft_shape_error_pct": 5.94}, "framework": "vllm", "id": "4b71b98dc1d4eb32", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10546.633855667491, "interactivity_tok_s": 98.30184675614467, "output_per_gpu": 24.249443329500483, "total_per_gpu": null, "tpot_error_pct": 2.95, "tpot_ms": 10.172748864837494, "tpot_relative": 0.970477, "ttft_error_pct": 84.16, "ttft_ms": 139.91176693873376, "ttft_relative": 1.841639, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9450.36590539999, "interactivity_tok_s": 98.2784127726709, "output_per_gpu": 24.374717582985518, "total_per_gpu": 48.577484151982105, "tpot_error_pct": 2.93, "tpot_ms": 10.17517450463016, "tpot_relative": 0.970709, "ttft_error_pct": 12.0, "ttft_ms": 85.08524890000129, "ttft_relative": 1.119965, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 9723.891526204534, "interactivity_tok_s": 95.39971862433688, "output_per_gpu": 23.6883670360648, "total_per_gpu": 47.20962490593205, "tpot_ms": 10.482211210053775, "tpot_relative": 1.0, "ttft_ms": 75.97133120289072, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11394.663900920219, "interactivity_tok_s": 91.08763261180538, "output_per_gpu": 44.88943284748328, "total_per_gpu": null, "tpot_error_pct": 0.87, "tpot_ms": 10.97843879927993, "tpot_relative": 1.04734, "ttft_error_pct": 73.61, "ttft_ms": 163.72100925685186, "ttft_relative": 2.155037, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10271.326155949902, "interactivity_tok_s": 91.82766025844745, "output_per_gpu": 45.38997238820151, "total_per_gpu": 89.9500399083589, "tpot_error_pct": 1.67, "tpot_ms": 10.8899649319771, "tpot_relative": 1.0389, "ttft_error_pct": 2.58, "ttft_ms": 96.73640090000008, "ttft_relative": 1.273328, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 10441.04007120186, "interactivity_tok_s": 90.29510269344651, "output_per_gpu": 44.64489753972604, "total_per_gpu": 88.4735130715084, "tpot_ms": 11.07479774838972, "tpot_relative": 1.056533, "ttft_ms": 94.30247800773941, "ttft_relative": 1.24129, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13263.713410112568, "interactivity_tok_s": 78.20209493793999, "output_per_gpu": 77.127721956058, "total_per_gpu": null, "tpot_error_pct": 4.42, "tpot_ms": 12.787381217774088, "tpot_relative": 1.219913, "ttft_error_pct": 77.73, "ttft_ms": 182.22242432967616, "ttft_relative": 2.398568, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11602.984065099947, "interactivity_tok_s": 79.73455984571079, "output_per_gpu": 76.81674106612441, "total_per_gpu": 154.3847205218192, "tpot_error_pct": 2.42, "tpot_ms": 12.541613096441939, "tpot_relative": 1.196466, "ttft_error_pct": 2.21, "ttft_ms": 104.79494329999939, "ttft_relative": 1.379401, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 11328.952307922009, "interactivity_tok_s": 81.66024382335291, "output_per_gpu": 78.53839116248693, "total_per_gpu": 157.8448577949494, "tpot_ms": 12.245861060165282, "tpot_relative": 1.168252, "ttft_ms": 102.52848400123185, "ttft_relative": 1.349568, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15539.66121077704, "interactivity_tok_s": 66.69205906466811, "output_per_gpu": 131.66310206177863, "total_per_gpu": null, "tpot_error_pct": 8.04, "tpot_ms": 14.99428888573297, "tpot_relative": 1.430451, "ttft_error_pct": 72.61, "ttft_ms": 200.50368067221035, "ttft_relative": 2.639202, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13905.957929637481, "interactivity_tok_s": 67.34664125976158, "output_per_gpu": 131.0253673291396, "total_per_gpu": 261.08293164636285, "tpot_error_pct": 6.99, "tpot_ms": 14.848550444303777, "tpot_relative": 1.416548, "ttft_error_pct": 3.37, "ttft_ms": 120.08028063749961, "ttft_relative": 1.5806, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 12999.678983044578, "interactivity_tok_s": 72.0521787165209, "output_per_gpu": 139.88733884878107, "total_per_gpu": 278.74141680597666, "tpot_ms": 13.878830839166689, "tpot_relative": 1.324037, "ttft_ms": 116.16326177609153, "ttft_relative": 1.529041, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20215.97527306218, "interactivity_tok_s": 51.15684053741447, "output_per_gpu": 202.41417714101564, "total_per_gpu": null, "tpot_error_pct": 19.92, "tpot_ms": 19.547727918588564, "tpot_relative": 1.864848, "ttft_error_pct": 54.16, "ttft_ms": 218.64961234607665, "ttft_relative": 2.878054, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17987.857598043483, "interactivity_tok_s": 51.300546871627816, "output_per_gpu": 199.80917928892708, "total_per_gpu": 401.7453951452148, "tpot_error_pct": 19.59, "tpot_ms": 19.49296958768013, "tpot_relative": 1.859624, "ttft_error_pct": 3.93, "ttft_ms": 147.40272265625117, "ttft_relative": 1.940241, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 15060.357572881549, "interactivity_tok_s": 61.34921450365477, "output_per_gpu": 237.91401682739513, "total_per_gpu": 478.3607091578899, "tpot_ms": 16.300127199516577, "tpot_relative": 1.555028, "ttft_ms": 141.83217131103447, "ttft_relative": 1.866917, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26706.04418341373, "interactivity_tok_s": 38.649131484797906, "output_per_gpu": 306.44748221763297, "total_per_gpu": null, "tpot_error_pct": 25.79, "tpot_ms": 25.87380263365908, "tpot_relative": 2.468353, "ttft_error_pct": 24.3, "ttft_ms": 237.1440891804935, "ttft_relative": 3.121494, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24234.231351015485, "interactivity_tok_s": 38.406199060668825, "output_per_gpu": 298.79758836748164, "total_per_gpu": 596.650001824791, "tpot_error_pct": 26.58, "tpot_ms": 26.037463338153763, "tpot_relative": 2.483967, "ttft_error_pct": 5.88, "ttft_ms": 202.01205324687513, "ttft_relative": 2.659056, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 19175.140812963582, "interactivity_tok_s": 48.6149029010346, "output_per_gpu": 375.71996451309457, "total_per_gpu": 750.2514285243986, "tpot_ms": 20.569824072994674, "tpot_relative": 1.962355, "ttft_ms": 190.79107900524832, "ttft_relative": 2.511356, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33817.2101132496, "interactivity_tok_s": 30.48106114690523, "output_per_gpu": 484.01390727341544, "total_per_gpu": null, "tpot_error_pct": 19.3, "tpot_ms": 32.807256780872635, "tpot_relative": 3.129803, "ttft_error_pct": 11.03, "ttft_ms": 255.38642641689717, "ttft_relative": 3.361616, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31202.785032970103, "interactivity_tok_s": 29.829107592348567, "output_per_gpu": 462.1455509259723, "total_per_gpu": 924.4994727723042, "tpot_error_pct": 21.9, "tpot_ms": 33.524301620626055, "tpot_relative": 3.198209, "ttft_error_pct": 8.15, "ttft_ms": 310.44299105937455, "ttft_relative": 4.086318, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 25622.485940304432, "interactivity_tok_s": 36.36269971767697, "output_per_gpu": 560.6427044113946, "total_per_gpu": 1121.538189869966, "tpot_ms": 27.50070835675248, "tpot_relative": 2.62356, "ttft_ms": 287.05532758203844, "ttft_relative": 3.778469, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45279.9432583738, "interactivity_tok_s": 22.730364274479964, "output_per_gpu": 722.9691038525319, "total_per_gpu": null, "tpot_error_pct": 20.46, "tpot_ms": 43.99401557865612, "tpot_relative": 4.197017, "ttft_error_pct": 42.29, "ttft_ms": 274.06532140859207, "ttft_relative": 3.607483, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43059.223038677774, "interactivity_tok_s": 21.62738819703907, "output_per_gpu": 669.0867245307538, "total_per_gpu": 1339.6433249841343, "tpot_error_pct": 26.61, "tpot_ms": 46.23766822370657, "tpot_relative": 4.411061, "ttft_error_pct": 12.73, "ttft_ms": 535.4225956507804, "ttft_relative": 7.047693, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 34053.25383745876, "interactivity_tok_s": 27.381528741854588, "output_per_gpu": 842.9995793530993, "total_per_gpu": 1687.8510931999588, "tpot_ms": 36.52097037487282, "tpot_relative": 3.48409, "ttft_ms": 474.94134097069036, "ttft_relative": 6.251586, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.93, "tpot_shape_error_pct": 1.71, "ttft_mape_pct": 67.75, "ttft_shape_error_pct": 3.74}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.97, "tpot_shape_error_pct": 2.11, "ttft_mape_pct": 8.35, "ttft_shape_error_pct": 5.2}, "framework": "vllm", "id": "6a0659d402a50199", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9912.804882512291, "interactivity_tok_s": 104.44925210913797, "output_per_gpu": 12.899981540602209, "total_per_gpu": null, "tpot_error_pct": 4.41, "tpot_ms": 9.57402738465863, "tpot_relative": 0.955898, "ttft_error_pct": 72.04, "ttft_ms": 118.57486800651307, "ttft_relative": 1.720426, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8879.277769799994, "interactivity_tok_s": 104.4268005215501, "output_per_gpu": 12.971212635303594, "total_per_gpu": 25.8509200805383, "tpot_error_pct": 4.39, "tpot_ms": 9.576085784545647, "tpot_relative": 0.956103, "ttft_error_pct": 5.05, "ttft_ms": 65.43859179999939, "ttft_relative": 0.949462, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 9287.52718246542, "interactivity_tok_s": 99.84279963380168, "output_per_gpu": 12.40035739857865, "total_per_gpu": 24.713236695257194, "tpot_ms": 10.015744787483413, "tpot_relative": 1.0, "ttft_ms": 68.92178789712489, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10330.955032155765, "interactivity_tok_s": 100.37250602480395, "output_per_gpu": 24.755697726295544, "total_per_gpu": null, "tpot_error_pct": 5.32, "tpot_ms": 9.962887643284317, "tpot_relative": 0.994723, "ttft_error_pct": 61.45, "ttft_ms": 138.9209730759083, "ttft_relative": 2.015632, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9340.852225849987, "interactivity_tok_s": 100.84959918807927, "output_per_gpu": 24.953157351379993, "total_per_gpu": 49.45007413530893, "tpot_error_pct": 5.77, "tpot_ms": 9.915755819069265, "tpot_relative": 0.990017, "ttft_error_pct": 11.11, "ttft_ms": 76.48412449999992, "ttft_relative": 1.109723, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 9916.817934100982, "interactivity_tok_s": 95.03112253078916, "output_per_gpu": 23.50106634008911, "total_per_gpu": 46.572441972436415, "tpot_ms": 10.522868438979133, "tpot_relative": 1.050633, "ttft_ms": 86.04675044771284, "ttft_relative": 1.24847, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11704.499159233295, "interactivity_tok_s": 88.57085568498529, "output_per_gpu": 43.70114372612812, "total_per_gpu": null, "tpot_error_pct": 2.05, "tpot_ms": 11.29039560774529, "tpot_relative": 1.127265, "ttft_error_pct": 69.77, "ttft_ms": 154.42445250986339, "ttft_relative": 2.240575, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10467.50227594997, "interactivity_tok_s": 88.28416892022314, "output_per_gpu": 42.54487050993771, "total_per_gpu": 85.50581360461125, "tpot_error_pct": 1.74, "tpot_ms": 11.327059111850929, "tpot_relative": 1.130925, "ttft_error_pct": 8.88, "ttft_ms": 82.88493922499909, "ttft_relative": 1.202594, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 10659.179773845244, "interactivity_tok_s": 86.75085914939382, "output_per_gpu": 41.74235058606181, "total_per_gpu": 83.89292541850452, "tpot_ms": 11.527263358601418, "tpot_relative": 1.150914, "ttft_ms": 90.96022846642882, "ttft_relative": 1.31976, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.65, "tpot_shape_error_pct": 20.91, "ttft_mape_pct": 92.5, "ttft_shape_error_pct": 67.32}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.33, "tpot_shape_error_pct": 4.99, "ttft_mape_pct": 17.04, "ttft_shape_error_pct": 8.04}, "framework": "vllm", "id": "e1d2fe51a70af017", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 124413.76774553738, "interactivity_tok_s": 8.251175701896921, "output_per_gpu": 4209.952077580959, "total_per_gpu": null, "tpot_error_pct": 1.99, "tpot_ms": 121.1948498163847, "tpot_relative": 1.01986, "ttft_error_pct": 88.69, "ttft_ms": 431.43638337582394, "ttft_relative": 0.113073, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 151861.04417902027, "interactivity_tok_s": 6.2006119938400985, "output_per_gpu": 3064.981210593699, "total_per_gpu": 6132.1897198821825, "tpot_error_pct": 35.71, "tpot_ms": 161.27440339654126, "tpot_relative": 1.357131, "ttft_error_pct": 13.57, "ttft_ms": 3297.8807003019365, "ttft_relative": 0.864326, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 113282.17565859134, "interactivity_tok_s": 8.415044177925754, "output_per_gpu": 4085.194071798601, "total_per_gpu": 8173.356823272174, "tpot_ms": 118.83478908205716, "tpot_relative": 1.0, "ttft_ms": 3815.5520446880937, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 223914.14733987403, "interactivity_tok_s": 4.579643516900018, "output_per_gpu": 4678.364509098862, "total_per_gpu": null, "tpot_error_pct": 23.31, "tpot_ms": 218.3576071608527, "tpot_relative": 1.837489, "ttft_error_pct": 96.3, "ttft_ms": 534.3152143217054, "ttft_relative": 0.140036, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 221666.14759946923, "interactivity_tok_s": 4.379606778300196, "output_per_gpu": 4197.292169612045, "total_per_gpu": 8395.464037246033, "tpot_error_pct": 28.95, "tpot_ms": 228.33100107405488, "tpot_relative": 1.921415, "ttft_error_pct": 20.51, "ttft_ms": 11493.196339652055, "ttft_relative": 3.012198, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 177407.53777225697, "interactivity_tok_s": 5.647351433000545, "output_per_gpu": 5190.432129121974, "total_per_gpu": 10381.952105525601, "tpot_ms": 177.07415801263159, "tpot_relative": 1.490087, "ttft_ms": 14459.23453838793, "ttft_relative": 3.789552, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.17, "tpot_shape_error_pct": 13.57, "ttft_mape_pct": 68.45, "ttft_shape_error_pct": 64.28}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.36, "tpot_shape_error_pct": 12.81, "ttft_mape_pct": 38.4, "ttft_shape_error_pct": 47.62}, "framework": "vllm", "id": "6398db7fe268f62d", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 48010.616034862964, "interactivity_tok_s": 21.440849206978818, "output_per_gpu": 1363.698394381306, "total_per_gpu": null, "tpot_error_pct": 17.0, "tpot_ms": 46.639943704958675, "tpot_relative": 0.830034, "ttft_error_pct": 44.79, "ttft_ms": 297.9536246902414, "ttft_relative": 0.552058, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56974.3823827412, "interactivity_tok_s": 16.319789719768554, "output_per_gpu": 1011.6973721577629, "total_per_gpu": 2022.416942086824, "tpot_error_pct": 9.05, "tpot_ms": 61.275299325007595, "tpot_relative": 1.090494, "ttft_error_pct": 16.8, "ttft_ms": 449.0235567859372, "ttft_relative": 0.831965, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 52356.37914548674, "interactivity_tok_s": 17.79662493081144, "output_per_gpu": 1097.9022896437634, "total_per_gpu": 2194.7434602856847, "tpot_ms": 56.190429583571884, "tpot_relative": 1.0, "ttft_ms": 539.7142353760955, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 61138.517813819984, "interactivity_tok_s": 16.822494907198255, "output_per_gpu": 2141.7594780225586, "total_per_gpu": null, "tpot_error_pct": 19.15, "tpot_ms": 59.44421475628477, "tpot_relative": 1.057906, "ttft_error_pct": 62.0, "ttft_ms": 327.08611814066325, "ttft_relative": 0.606036, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 76788.16780347975, "interactivity_tok_s": 12.13816388590108, "output_per_gpu": 1506.6344562687923, "total_per_gpu": 3011.881032440628, "tpot_error_pct": 12.04, "tpot_ms": 82.38478318467396, "tpot_relative": 1.466171, "ttft_error_pct": 9.57, "ttft_ms": 778.3832823755846, "ttft_relative": 1.442214, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 68685.93399389557, "interactivity_tok_s": 13.60017624269883, "output_per_gpu": 1680.1184386079285, "total_per_gpu": 3358.689187308878, "tpot_ms": 73.52845890779128, "tpot_relative": 1.308558, "ttft_ms": 860.7926147137732, "ttft_relative": 1.594904, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 117287.7152075924, "interactivity_tok_s": 8.752140523395088, "output_per_gpu": 4465.736237362516, "total_per_gpu": null, "tpot_error_pct": 3.36, "tpot_ms": 114.25776326681793, "tpot_relative": 2.033403, "ttft_error_pct": 98.56, "ttft_ms": 402.0233856376399, "ttft_relative": 0.744882, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 139724.3464412221, "interactivity_tok_s": 6.751598167234799, "output_per_gpu": 3324.095876143256, "total_per_gpu": 6646.73427968656, "tpot_error_pct": 33.99, "tpot_ms": 148.1130800782776, "tpot_relative": 2.635913, "ttft_error_pct": 88.81, "ttft_ms": 3129.1381893778776, "ttft_relative": 5.797768, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 129969.44389417689, "interactivity_tok_s": 9.046564297010606, "output_per_gpu": 3528.6403652544914, "total_per_gpu": 7055.733573977081, "tpot_ms": 110.53920219529586, "tpot_relative": 1.967225, "ttft_ms": 27971.89732845201, "ttft_relative": 51.827237, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp4"], "rows": 3, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "2d5d35eb35c7058f", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 38837.34995902036, "interactivity_tok_s": 29.644657156476647, "output_per_gpu": 186.0532516850564, "total_per_gpu": 248.34348829371888, "tpot_ms": 33.73289138483168, "tpot_relative": 1.0, "ttft_ms": 7838.549095116503, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "a167063969d43b6d", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 76796.64190346646, "interactivity_tok_s": 14.54206438334768, "output_per_gpu": 2936.524467130476, "total_per_gpu": 2350.0731537642177, "tpot_ms": 68.76602754868242, "tpot_relative": 1.0, "ttft_ms": 13516.949180569345, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 120781.03929815718, "interactivity_tok_s": 10.515752457549652, "output_per_gpu": 3595.7307319897486, "total_per_gpu": 2876.8860330092807, "tpot_ms": 95.09542983603257, "tpot_relative": 1.382884, "ttft_ms": 33277.27918209851, "ttft_relative": 2.461893, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 21}, {"aic": {"coverage_pct": 60.76, "points": 48, "status_counts": {"failed": 31, "success": 48, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.4, "tpot_shape_error_pct": 17.22, "ttft_mape_pct": 42.38, "ttft_shape_error_pct": 50.21}, "aisimulate": {"coverage_pct": 53.16, "points": 42, "status_counts": {"failed": 37, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.76, "tpot_shape_error_pct": 15.85, "ttft_mape_pct": 15.47, "ttft_shape_error_pct": 36.42}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 74.29, "points": 26, "status_counts": {"failed": 9, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.09, "tpot_shape_error_pct": 17.31, "ttft_mape_pct": 39.62, "ttft_shape_error_pct": 47.21}, "aisimulate": {"coverage_pct": 65.71, "points": 23, "status_counts": {"failed": 12, "success": 23, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.56, "tpot_shape_error_pct": 19.25, "ttft_mape_pct": 14.89, "ttft_shape_error_pct": 49.41}, "gpu": "b200", "precisions": ["fp4"], "rows": 35, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "088a412b22ec99b6", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (340570279288 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "28481566954", "measured": {"e2e_ms": 34425.50472061797, "interactivity_tok_s": 29.41387660908234, "output_per_gpu": 821.1104465697387, "total_per_gpu": 3694.2471765621804, "tpot_ms": 33.99755881518938, "tpot_relative": 1.0, "ttft_ms": 3113.6289925519523, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (340570279288 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "28481566954", "measured": {"e2e_ms": 52376.73626548671, "interactivity_tok_s": 25.978579584597814, "output_per_gpu": 1077.4934838718157, "total_per_gpu": 4849.0685109294045, "tpot_ms": 38.49325159382002, "tpot_relative": 1.132236, "ttft_ms": 16923.054857430077, "ttft_relative": 5.435155, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.39, "tpot_shape_error_pct": 15.28, "ttft_mape_pct": 48.47, "ttft_shape_error_pct": 49.5}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "0452cc3d7d742cf6", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 34857.035, "interactivity_tok_s": 30.91381229133177, "output_per_gpu": 303.66379166666667, "total_per_gpu": null, "tpot_error_pct": 6.42, "tpot_ms": 32.348, "tpot_relative": 0.935753, "ttft_error_pct": 23.08, "ttft_ms": 1765.031, "ttft_relative": 0.769151, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 34133.195892829695, "interactivity_tok_s": 28.927698180207884, "output_per_gpu": 833.7512268352299, "total_per_gpu": 2500.746096182814, "tpot_ms": 34.56894474528888, "tpot_relative": 1.0, "ttft_ms": 2294.7788264558694, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 43198.577, "interactivity_tok_s": 24.69013875857982, "output_per_gpu": 485.0534583333333, "total_per_gpu": null, "tpot_error_pct": 0.99, "tpot_ms": 40.502, "tpot_relative": 1.17163, "ttft_error_pct": 40.03, "ttft_ms": 1765.031, "ttft_relative": 0.769151, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 39882.47604638553, "interactivity_tok_s": 24.933466697274213, "output_per_gpu": 1420.615825531001, "total_per_gpu": 4262.153209475849, "tpot_ms": 40.10673734789244, "tpot_relative": 1.160196, "ttft_ms": 2943.394852349614, "ttft_relative": 1.282649, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 52155.965, "interactivity_tok_s": 20.301270859555807, "output_per_gpu": 626.5344166666666, "total_per_gpu": null, "tpot_error_pct": 14.77, "tpot_ms": 49.258, "tpot_relative": 1.424921, "ttft_error_pct": 82.29, "ttft_ms": 1765.031, "ttft_relative": 0.769151, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 49484.05077249043, "interactivity_tok_s": 23.298932737534596, "output_per_gpu": 2270.7556117764007, "total_per_gpu": 6811.6907271188, "tpot_ms": 42.92042091649114, "tpot_relative": 1.241589, "ttft_ms": 9965.269630730952, "ttft_relative": 4.342584, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b78b2d2a042fff3e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33012183714", "measured": {"e2e_ms": 27310.845500319374, "interactivity_tok_s": 39.91513090058221, "output_per_gpu": 263.7703116974889, "total_per_gpu": 1188.9265927071015, "tpot_ms": 25.05315596961787, "tpot_relative": 1.0, "ttft_ms": 4247.355013934248, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "26230fa62074acac", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: DSV4 module data missing for ModuleKey { fmha_quant: \"bfloat16\", kv_quant: \"bfloat16\", gemm_quant: \"fp8_block\" }", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28481566954", "measured": {"e2e_ms": 16742.130857100165, "interactivity_tok_s": 89.88848024226446, "output_per_gpu": 6.945582666518074, "total_per_gpu": 31.0904975794703, "tpot_ms": 11.124896063487034, "tpot_relative": 1.0, "ttft_ms": 6403.202616500039, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 17450.319365018688, "interactivity_tok_s": 64.36743462300605, "output_per_gpu": 103.44746886406566, "total_per_gpu": 466.8502755835327, "tpot_ms": 15.535806357001876, "tpot_relative": 1.39649, "ttft_ms": 3251.528011556212, "ttft_relative": 0.507797, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: DSV4 module data missing for ModuleKey { fmha_quant: \"bfloat16\", kv_quant: \"bfloat16\", gemm_quant: \"fp8_block\" }", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28481566954", "measured": {"e2e_ms": 22504.17046619061, "interactivity_tok_s": 48.98151873031743, "output_per_gpu": 160.5947965348893, "total_per_gpu": 718.131668640771, "tpot_ms": 20.4158634914079, "tpot_relative": 1.835151, "ttft_ms": 3635.940268787442, "ttft_relative": 0.567832, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: DSV4 module data missing for ModuleKey { fmha_quant: \"bfloat16\", kv_quant: \"bfloat16\", gemm_quant: \"fp8_block\" }", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28481566954", "measured": {"e2e_ms": 31267.67551119216, "interactivity_tok_s": 34.02849290256764, "output_per_gpu": 456.7655398055376, "total_per_gpu": 2062.1065521834566, "tpot_ms": 29.387137504539446, "tpot_relative": 2.641565, "ttft_ms": 4262.39772356484, "ttft_relative": 0.665667, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cc205e6d70677422", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 246522.91340256293, "interactivity_tok_s": 18.43995826688343, "output_per_gpu": 3628.0595635398413, "total_per_gpu": 8164.477728760442, "tpot_ms": 54.230057656687514, "tpot_relative": 1.0, "ttft_ms": 196599.6096490632, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12345, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28078051095", "measured": {"e2e_ms": 369677.8667333677, "interactivity_tok_s": 18.455947111470653, "output_per_gpu": 3636.828279751049, "total_per_gpu": 8184.275689069177, "tpot_ms": 54.18307681313655, "tpot_relative": 0.999134, "ttft_ms": 319800.70535820094, "ttft_relative": 1.62666, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.16, "tpot_shape_error_pct": 14.71, "ttft_mape_pct": 36.43, "ttft_shape_error_pct": 45.42}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.53, "tpot_shape_error_pct": 15.06, "ttft_mape_pct": 14.97, "ttft_shape_error_pct": 60.29}, "framework": "vllm", "id": "e89c5adab28c0d03", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 11093.478670446708, "interactivity_tok_s": 95.55347797274197, "output_per_gpu": 11.52704249034724, "total_per_gpu": null, "tpot_error_pct": 8.15, "tpot_ms": 10.465343818100107, "tpot_relative": 1.081456, "ttft_error_pct": 13.47, "ttft_ms": 387.43194453029724, "ttft_relative": 0.865259, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10019.738467900013, "interactivity_tok_s": 95.60419805301554, "output_per_gpu": 11.605841846326365, "total_per_gpu": 103.88993718098187, "tpot_error_pct": 8.09, "tpot_ms": 10.459791728449712, "tpot_relative": 1.080882, "ttft_error_pct": 33.13, "ttft_ms": 299.4057854999992, "ttft_relative": 0.668668, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 9440.680929407245, "interactivity_tok_s": 103.33687613270234, "output_per_gpu": 12.317310061653332, "total_per_gpu": 110.25865986179113, "tpot_ms": 9.677087574389493, "tpot_relative": 1.0, "ttft_ms": 447.76437021791935, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11404.509754228195, "interactivity_tok_s": 93.30689850025642, "output_per_gpu": 22.42533923084079, "total_per_gpu": null, "tpot_error_pct": 3.09, "tpot_ms": 10.717321184963101, "tpot_relative": 1.107494, "ttft_error_pct": 10.95, "ttft_ms": 440.69018201094076, "ttft_relative": 0.984201, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10012.039041, "interactivity_tok_s": 96.4285984206914, "output_per_gpu": 23.277716892933523, "total_per_gpu": 207.43861807891986, "tpot_error_pct": 0.25, "tpot_ms": 10.370367467514933, "tpot_relative": 1.071641, "ttft_error_pct": 19.26, "ttft_ms": 320.71042975000046, "ttft_relative": 0.716248, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 10102.744298637845, "interactivity_tok_s": 96.18774604211828, "output_per_gpu": 23.074309589654334, "total_per_gpu": 205.625960501999, "tpot_ms": 10.396334680325333, "tpot_relative": 1.074325, "ttft_ms": 397.2004056442529, "ttft_relative": 0.887075, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13488.877919754068, "interactivity_tok_s": 78.69840005577814, "output_per_gpu": 37.92012968335366, "total_per_gpu": null, "tpot_error_pct": 11.76, "tpot_ms": 12.706738628628305, "tpot_relative": 1.313075, "ttft_error_pct": 50.86, "ttft_ms": 489.8843026673136, "ttft_relative": 1.094067, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11634.105244325026, "interactivity_tok_s": 81.44278404220132, "output_per_gpu": 38.63836240087859, "total_per_gpu": 347.3358345915687, "tpot_error_pct": 7.99, "tpot_ms": 12.27855864408845, "tpot_relative": 1.268828, "ttft_error_pct": 7.1, "ttft_ms": 347.7976969000004, "ttft_relative": 0.776743, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 10774.293169740122, "interactivity_tok_s": 87.95013170459005, "output_per_gpu": 41.65940319392897, "total_per_gpu": 374.493189613568, "tpot_ms": 11.370079619196417, "tpot_relative": 1.174949, "ttft_ms": 324.7347128984984, "ttft_relative": 0.725236, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16189.331288916019, "interactivity_tok_s": 65.36639850034837, "output_per_gpu": 63.189762550624565, "total_per_gpu": null, "tpot_error_pct": 11.21, "tpot_ms": 15.298379946612332, "tpot_relative": 1.580887, "ttft_error_pct": 24.66, "ttft_ms": 539.0886035316033, "ttft_relative": 1.203956, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14074.968217999984, "interactivity_tok_s": 67.83523056822072, "output_per_gpu": 64.75309941315575, "total_per_gpu": 576.0087571742731, "tpot_error_pct": 7.16, "tpot_ms": 14.741602433182816, "tpot_relative": 1.523351, "ttft_error_pct": 2.81, "ttft_ms": 420.28036088750014, "ttft_relative": 0.938619, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 13176.411400231882, "interactivity_tok_s": 72.69300298256834, "output_per_gpu": 69.11871814150224, "total_per_gpu": 614.8429541594555, "tpot_ms": 13.756482178068751, "tpot_relative": 1.421552, "ttft_ms": 432.43409764545504, "ttft_relative": 0.965763, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23325.67478877076, "interactivity_tok_s": 44.992558652787636, "output_per_gpu": 87.71450423311943, "total_per_gpu": null, "tpot_error_pct": 34.62, "tpot_ms": 22.225897569353776, "tpot_relative": 2.296755, "ttft_error_pct": 9.48, "ttft_ms": 588.5815753218474, "ttft_relative": 1.31449, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20283.18646366874, "interactivity_tok_s": 46.373576586922894, "output_per_gpu": 88.06138410929802, "total_per_gpu": 794.7319500792532, "tpot_error_pct": 30.61, "tpot_ms": 21.564004193758795, "tpot_relative": 2.228357, "ttft_error_pct": 2.25, "ttft_ms": 549.7220481937509, "ttft_relative": 1.227704, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 15649.16467731091, "interactivity_tok_s": 60.56957640725709, "output_per_gpu": 113.50812111693044, "total_per_gpu": 1024.3823823292084, "tpot_ms": 16.509938806179036, "tpot_relative": 1.706086, "ttft_ms": 537.6010043255519, "ttft_relative": 1.200634, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32081.24277825583, "interactivity_tok_s": 32.535812706842385, "output_per_gpu": 127.55116839717613, "total_per_gpu": null, "tpot_error_pct": 37.02, "tpot_ms": 30.735362568327574, "tpot_relative": 3.176096, "ttft_error_pct": 14.34, "ttft_ms": 638.9668708567239, "ttft_relative": 1.427016, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28428.683181853074, "interactivity_tok_s": 33.509442232398776, "output_per_gpu": 127.62943657516962, "total_per_gpu": 1141.3030599593678, "tpot_error_pct": 33.04, "tpot_ms": 29.84233497724844, "tpot_relative": 3.083814, "ttft_error_pct": 6.99, "ttft_ms": 798.1104222625006, "ttft_relative": 1.782434, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 21522.606272045232, "interactivity_tok_s": 44.57979343326533, "output_per_gpu": 167.97968276302245, "total_per_gpu": 1502.1278091714132, "tpot_ms": 22.431687609700823, "tpot_relative": 2.318021, "ttft_ms": 745.9534745932615, "ttft_relative": 1.665951, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45591.06917019474, "interactivity_tok_s": 22.78341260721732, "output_per_gpu": 179.50884129188853, "total_per_gpu": null, "tpot_error_pct": 27.6, "tpot_ms": 43.89158100412141, "tpot_relative": 4.535619, "ttft_error_pct": 44.4, "ttft_ms": 689.9818029785396, "ttft_relative": 1.540948, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 40766.586303940516, "interactivity_tok_s": 23.367315958771346, "output_per_gpu": 178.18776182421394, "total_per_gpu": 1606.1449231094095, "tpot_error_pct": 24.41, "tpot_ms": 42.794816561918054, "tpot_relative": 4.422283, "ttft_error_pct": 3.81, "ttft_ms": 1288.3481729750004, "ttft_relative": 2.877291, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 32999.00907169649, "interactivity_tok_s": 29.071170909968014, "output_per_gpu": 220.11364398178418, "total_per_gpu": 1984.055516322294, "tpot_ms": 34.398339272159035, "tpot_relative": 3.554617, "ttft_ms": 1241.0579864164902, "ttft_relative": 2.771677, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71278.99857641118, "interactivity_tok_s": 14.50347669424871, "output_per_gpu": 229.632855776635, "total_per_gpu": null, "tpot_error_pct": 23.46, "tpot_ms": 68.94898520411631, "tpot_relative": 7.124973, "ttft_error_pct": 65.68, "ttft_ms": 744.1867126001886, "ttft_relative": 1.662005, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 64167.140137289956, "interactivity_tok_s": 14.891988242039085, "output_per_gpu": 226.69021662809618, "total_per_gpu": 2046.5778292171663, "tpot_error_pct": 20.24, "tpot_ms": 67.15020074868626, "tpot_relative": 6.939092, "ttft_error_pct": 6.84, "ttft_ms": 2316.6102004398444, "ttft_relative": 5.173726, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 53688.76278363223, "interactivity_tok_s": 17.906623613722452, "output_per_gpu": 270.9992646215267, "total_per_gpu": 2446.603540983305, "tpot_ms": 55.84525712785218, "tpot_relative": 5.770874, "ttft_ms": 2168.2768790245973, "ttft_relative": 4.842451, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 118613.19505930718, "interactivity_tok_s": 8.683462446500592, "output_per_gpu": 275.9895303691283, "total_per_gpu": null, "tpot_error_pct": 33.53, "tpot_ms": 115.16143544825219, "tpot_relative": 11.900423, "ttft_error_pct": 94.03, "ttft_ms": 803.0465957451898, "ttft_relative": 1.793458, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 142019.95106869476, "interactivity_tok_s": 7.578557344934074, "output_per_gpu": 205.72276478221838, "total_per_gpu": 1850.9725470371077, "tpot_error_pct": 53.0, "tpot_ms": 131.95123484398454, "tpot_relative": 13.635428, "ttft_error_pct": 52.49, "ttft_ms": 20509.070440694253, "ttft_relative": 45.803266, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 92870.57697570688, "interactivity_tok_s": 11.594934723510653, "output_per_gpu": 313.96259990428126, "total_per_gpu": 2824.8509776466426, "tpot_ms": 86.24455625199288, "tpot_relative": 8.912243, "ttft_ms": 13449.14228732498, "ttft_relative": 30.036205, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.26, "tpot_shape_error_pct": 23.06, "ttft_mape_pct": 35.72, "ttft_shape_error_pct": 41.02}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.54, "tpot_shape_error_pct": 20.91, "ttft_mape_pct": 18.69, "ttft_shape_error_pct": 56.75}, "framework": "vllm", "id": "8e538420e3b21dce", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10286.602271174026, "interactivity_tok_s": 103.1076841904319, "output_per_gpu": 12.431218455713212, "total_per_gpu": null, "tpot_error_pct": 8.43, "tpot_ms": 9.69859819713415, "tpot_relative": 0.91571, "ttft_error_pct": 17.91, "ttft_ms": 364.9363155057916, "ttft_relative": 0.820902, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9286.7673018, "interactivity_tok_s": 103.16674334168425, "output_per_gpu": 12.521849231374697, "total_per_gpu": 112.08959653788662, "tpot_error_pct": 8.48, "tpot_ms": 9.693046107775631, "tpot_relative": 0.915186, "ttft_error_pct": 37.25, "ttft_ms": 278.97132469999934, "ttft_relative": 0.627529, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 10287.101517384872, "interactivity_tok_s": 94.41673069017322, "output_per_gpu": 11.30380217182651, "total_per_gpu": 101.1862226874153, "tpot_ms": 10.591343215234614, "tpot_relative": 1.0, "ttft_ms": 444.5555373094976, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11202.717298885575, "interactivity_tok_s": 94.83343481489531, "output_per_gpu": 22.829282679965647, "total_per_gpu": null, "tpot_error_pct": 5.64, "tpot_ms": 10.544804181688585, "tpot_relative": 0.995606, "ttft_error_pct": 7.53, "ttft_ms": 415.38262101815275, "ttft_relative": 0.934377, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9847.487112350016, "interactivity_tok_s": 97.86265665982015, "output_per_gpu": 23.671106232517108, "total_per_gpu": 210.94429440214086, "tpot_error_pct": 8.56, "tpot_ms": 10.21840234192798, "tpot_relative": 0.964788, "ttft_error_pct": 22.75, "ttft_ms": 298.4178513000017, "ttft_relative": 0.671272, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 10818.741091003176, "interactivity_tok_s": 89.4861084662072, "output_per_gpu": 21.544884840161743, "total_per_gpu": 191.99654151947183, "tpot_ms": 11.174918846511599, "tpot_relative": 1.055099, "ttft_ms": 386.2910586525686, "ttft_relative": 0.868938, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12719.633901375415, "interactivity_tok_s": 83.45659386307727, "output_per_gpu": 40.2134215470376, "total_per_gpu": null, "tpot_error_pct": 3.37, "tpot_ms": 11.982276698718929, "tpot_relative": 1.131327, "ttft_error_pct": 27.27, "ttft_ms": 461.7648385859507, "ttft_relative": 1.038711, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10974.64439934997, "interactivity_tok_s": 86.29890317929865, "output_per_gpu": 40.94569814566642, "total_per_gpu": 368.0774068736439, "tpot_error_pct": 6.55, "tpot_ms": 11.587632787433614, "tpot_relative": 1.094066, "ttft_error_pct": 10.86, "ttft_ms": 323.43647272499976, "ttft_relative": 0.72755, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 11771.461716893828, "interactivity_tok_s": 80.64473889053879, "output_per_gpu": 38.146526246552995, "total_per_gpu": 342.91452088855567, "tpot_ms": 12.40006494853094, "tpot_relative": 1.170774, "ttft_ms": 362.82141560222954, "ttft_relative": 0.816144, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16210.42333276106, "interactivity_tok_s": 65.14983237426983, "output_per_gpu": 63.107543769848995, "total_per_gpu": null, "tpot_error_pct": 4.7, "tpot_ms": 15.349233659654638, "tpot_relative": 1.449224, "ttft_error_pct": 39.04, "ttft_ms": 508.15729893436577, "ttft_relative": 1.143068, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14108.075331412512, "interactivity_tok_s": 67.52306344694881, "output_per_gpu": 64.6408929391834, "total_per_gpu": 575.0106287108409, "tpot_error_pct": 8.05, "tpot_ms": 14.809754607559759, "tpot_relative": 1.398289, "ttft_error_pct": 53.11, "ttft_ms": 390.87843081250037, "ttft_relative": 0.879257, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 15745.839816977968, "interactivity_tok_s": 62.09057479604015, "output_per_gpu": 57.92759188951615, "total_per_gpu": 515.2927120520139, "tpot_ms": 16.105503988083154, "tpot_relative": 1.520629, "ttft_ms": 833.5729855811223, "ttft_relative": 1.875071, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22834.859367158788, "interactivity_tok_s": 45.915576519569136, "output_per_gpu": 89.59985113560926, "total_per_gpu": null, "tpot_error_pct": 22.79, "tpot_ms": 21.779101468404775, "tpot_relative": 2.056312, "ttft_error_pct": 7.01, "ttft_ms": 554.8385649807041, "ttft_relative": 1.248075, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19868.883078112653, "interactivity_tok_s": 47.27419432090946, "output_per_gpu": 89.92945422724871, "total_per_gpu": 811.5908153212655, "tpot_error_pct": 19.26, "tpot_ms": 21.153189691858973, "tpot_relative": 1.997215, "ttft_error_pct": 1.19, "ttft_ms": 512.3431032687497, "ttft_relative": 1.152484, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 16750.35628147598, "interactivity_tok_s": 56.38135807744487, "output_per_gpu": 106.00681333842573, "total_per_gpu": 956.6849571836243, "tpot_ms": 17.73635886220424, "tpot_relative": 1.674609, "ttft_ms": 518.493894416315, "ttft_relative": 1.16632, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31314.716687436132, "interactivity_tok_s": 33.309125615485925, "output_per_gpu": 130.67338404635044, "total_per_gpu": null, "tpot_error_pct": 27.76, "tpot_ms": 30.02180277992901, "tpot_relative": 2.83456, "ttft_error_pct": 17.06, "ttft_ms": 602.4124435687519, "ttft_relative": 1.355089, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27763.772181137418, "interactivity_tok_s": 34.264348153631644, "output_per_gpu": 130.66765384120242, "total_per_gpu": 1168.471766141835, "tpot_error_pct": 24.19, "tpot_ms": 29.184854050521636, "tpot_relative": 2.755539, "ttft_error_pct": 2.47, "ttft_ms": 744.2791120874997, "ttft_relative": 1.67421, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 22486.567374999868, "interactivity_tok_s": 42.55417182929338, "output_per_gpu": 160.4872481197295, "total_per_gpu": 1435.1280729475563, "tpot_ms": 23.49945861974504, "tpot_relative": 2.218742, "ttft_ms": 726.3667413011717, "ttft_relative": 1.633917, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 42017.849315586274, "interactivity_tok_s": 24.729717997125277, "output_per_gpu": 194.774366925158, "total_per_gpu": null, "tpot_error_pct": 17.06, "tpot_ms": 40.43717765468436, "tpot_relative": 3.817946, "ttft_error_pct": 45.39, "ttft_ms": 650.6165748441691, "ttft_relative": 1.463521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37464.074289556265, "interactivity_tok_s": 25.43818074449506, "output_per_gpu": 193.72387880772328, "total_per_gpu": 1746.184032206668, "tpot_error_pct": 13.8, "tpot_ms": 39.31098729284737, "tpot_relative": 3.711615, "ttft_error_pct": 0.6, "ttft_ms": 1198.4516471046895, "ttft_relative": 2.695842, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 33080.49059974874, "interactivity_tok_s": 28.94780532032795, "output_per_gpu": 219.3063289073109, "total_per_gpu": 1976.7785574843792, "tpot_ms": 34.54493316278358, "tpot_relative": 3.26162, "ttft_ms": 1191.338586098027, "ttft_relative": 2.679842, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 64797.44631773342, "interactivity_tok_s": 15.960575207500737, "output_per_gpu": 252.60254732477773, "total_per_gpu": null, "tpot_error_pct": 17.94, "tpot_ms": 62.65438350430165, "tpot_relative": 5.915622, "ttft_error_pct": 66.16, "ttft_ms": 702.0119928328346, "ttft_relative": 1.579132, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 57910.96493561764, "interactivity_tok_s": 16.519517002888527, "output_per_gpu": 251.0314951130733, "total_per_gpu": 2266.3328835955504, "tpot_error_pct": 13.95, "tpot_ms": 60.53445750412343, "tpot_relative": 5.715466, "ttft_error_pct": 3.63, "ttft_ms": 2149.599004716404, "ttft_relative": 4.835389, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 51045.904130380586, "interactivity_tok_s": 18.823812655412343, "output_per_gpu": 284.6311489778428, "total_per_gpu": 2569.6733086559825, "tpot_ms": 53.12420062321825, "tpot_relative": 5.015813, "ttft_ms": 2074.23922054968, "ttft_relative": 4.665872, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 109595.3192231032, "interactivity_tok_s": 9.399355108470143, "output_per_gpu": 298.6988881647338, "total_per_gpu": null, "tpot_error_pct": 29.68, "tpot_ms": 106.39027767967391, "tpot_relative": 10.045022, "ttft_error_pct": 94.08, "ttft_ms": 758.0651567967894, "ttft_relative": 1.70522, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 120978.87381857967, "interactivity_tok_s": 8.895441893158884, "output_per_gpu": 241.36566224836506, "total_per_gpu": 2171.6663933235754, "tpot_error_pct": 37.03, "tpot_ms": 112.4171246365016, "tpot_relative": 10.614057, "ttft_error_pct": 36.39, "ttft_ms": 17456.000283114445, "ttft_relative": 39.266186, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 88349.05639949057, "interactivity_tok_s": 12.1891371731923, "output_per_gpu": 329.7263808946419, "total_per_gpu": 2966.6842156042967, "tpot_ms": 82.04026140581227, "tpot_relative": 7.745973, "ttft_ms": 12799.058217667243, "ttft_relative": 28.790685, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.5, "tpot_shape_error_pct": 12.01, "ttft_mape_pct": 47.07, "ttft_shape_error_pct": 62.03}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.42, "tpot_shape_error_pct": 24.3, "ttft_mape_pct": 7.92, "ttft_shape_error_pct": 13.01}, "framework": "vllm", "id": "7f7ce2b59921715b", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 32140.188202266832, "interactivity_tok_s": 33.17495184281014, "output_per_gpu": 254.6344765779183, "total_per_gpu": null, "tpot_error_pct": 36.89, "tpot_ms": 30.14322386173186, "tpot_relative": 0.631099, "ttft_error_pct": 26.36, "ttft_ms": 1303.6701917151377, "ttft_relative": 1.263579, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 57040.81128773759, "interactivity_tok_s": 16.514999804506868, "output_per_gpu": 127.56243772554761, "total_per_gpu": 1149.8194917250871, "tpot_error_pct": 26.77, "tpot_ms": 60.551014946249325, "tpot_relative": 1.267736, "ttft_error_pct": 13.59, "ttft_ms": 1171.965930673436, "ttft_relative": 1.135925, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 45069.108867611794, "interactivity_tok_s": 20.936662574199314, "output_per_gpu": 160.82267313299127, "total_per_gpu": 1449.6198691145855, "tpot_ms": 47.763104384760965, "tpot_relative": 1.0, "ttft_ms": 1031.7282624597283, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 42135.9942351427, "interactivity_tok_s": 25.127395743394136, "output_per_gpu": 388.45647995529174, "total_per_gpu": null, "tpot_error_pct": 37.31, "tpot_ms": 39.79720024359846, "tpot_relative": 0.833221, "ttft_error_pct": 0.46, "ttft_ms": 1423.4583859414752, "ttft_relative": 1.379683, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 81127.72723525595, "interactivity_tok_s": 11.571951496780965, "output_per_gpu": 179.480741366557, "total_per_gpu": 1620.3668226885877, "tpot_error_pct": 36.14, "tpot_ms": 86.41584786093993, "tpot_relative": 1.809259, "ttft_error_pct": 7.28, "ttft_ms": 1520.1539166171863, "ttft_relative": 1.473405, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 59873.17865060231, "interactivity_tok_s": 15.753498719328094, "output_per_gpu": 242.06570211419378, "total_per_gpu": 2185.3889705948377, "tpot_ms": 63.47796243973994, "tpot_relative": 1.329017, "ttft_ms": 1417.0057830313453, "ttft_relative": 1.373429, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 60056.30195343854, "interactivity_tok_s": 17.4836998115422, "output_per_gpu": 545.088507537146, "total_per_gpu": null, "tpot_error_pct": 45.65, "tpot_ms": 57.19613187020238, "tpot_relative": 1.197496, "ttft_error_pct": 59.59, "ttft_ms": 1544.659050221507, "ttft_relative": 1.497157, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 74975.8862318043, "interactivity_tok_s": 12.878570892707526, "output_per_gpu": 388.3745134710806, "total_per_gpu": 3494.365648667381, "tpot_error_pct": 26.22, "tpot_ms": 77.64836706891514, "tpot_relative": 1.625698, "ttft_error_pct": 13.71, "ttft_ms": 3298.4752159304708, "ttft_relative": 3.197039, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 101007.46226955699, "interactivity_tok_s": 9.501733878951795, "output_per_gpu": 289.1201045280685, "total_per_gpu": 2601.3328026408562, "tpot_ms": 105.24394944539503, "tpot_relative": 2.203457, "ttft_ms": 3822.6118238670097, "ttft_relative": 3.705057, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 95503.18589105339, "interactivity_tok_s": 10.902236249783732, "output_per_gpu": 685.5478106739613, "total_per_gpu": null, "tpot_error_pct": 35.11, "tpot_ms": 91.7243010597791, "tpot_relative": 1.920401, "ttft_error_pct": 69.65, "ttft_ms": 1669.2259068993724, "ttft_relative": 1.617893, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 122621.29186158502, "interactivity_tok_s": 7.874078095303013, "output_per_gpu": 476.232255978805, "total_per_gpu": 4286.1249016454885, "tpot_error_pct": 10.15, "tpot_ms": 126.99899440882007, "tpot_relative": 2.658935, "ttft_error_pct": 1.67, "ttft_ms": 5408.296377196098, "ttft_relative": 5.241978, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 135912.89189674935, "interactivity_tok_s": 7.074876627569321, "output_per_gpu": 429.9849095084528, "total_per_gpu": 3869.89542358505, "tpot_ms": 141.34522093335286, "tpot_relative": 2.959297, "ttft_ms": 5500.087769584161, "ttft_relative": 5.330946, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 159046.42992168182, "interactivity_tok_s": 6.505649306710892, "output_per_gpu": 823.3067542885426, "total_per_gpu": null, "tpot_error_pct": 17.53, "tpot_ms": 153.71255855560054, "tpot_relative": 3.218228, "ttft_error_pct": 79.29, "ttft_ms": 1798.4825193024656, "ttft_relative": 1.743175, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 185617.77429571064, "interactivity_tok_s": 5.216887121390001, "output_per_gpu": 630.4324667427422, "total_per_gpu": 5672.728571425185, "tpot_error_pct": 2.84, "tpot_ms": 191.6851901778464, "tpot_relative": 4.013248, "ttft_error_pct": 3.37, "ttft_ms": 8975.800578521877, "ttft_relative": 8.699772, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "33554397154", "measured": {"e2e_ms": 180370.29367174336, "interactivity_tok_s": 5.365181878172914, "output_per_gpu": 648.6259166719586, "total_per_gpu": 5836.436039981646, "tpot_ms": 186.38697116090034, "tpot_relative": 3.902321, "ttft_ms": 8683.175162906446, "ttft_relative": 8.416145, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 84.62, "points": 22, "status_counts": {"failed": 4, "success": 22, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.59, "tpot_shape_error_pct": 17.09, "ttft_mape_pct": 45.65, "ttft_shape_error_pct": 54.92}, "aisimulate": {"coverage_pct": 69.23, "points": 18, "status_counts": {"failed": 8, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.72, "tpot_shape_error_pct": 9.67, "ttft_mape_pct": 14.13, "ttft_shape_error_pct": 12.78}, "gpu": "b300", "precisions": ["fp4"], "rows": 26, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "b53d7c2e217238a9", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"w4a8_mxfp4_mxfp8\", distribution: \"power_law_1.01\", topk: 6, num_experts: 384, hidden_size: 7168, inter_size: 3072, moe_tp_size: 4, moe_ep_size: 1 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30431262611", "measured": {"e2e_ms": 9851.63215749999, "interactivity_tok_s": 97.71338365137588, "output_per_gpu": 23.606060765421272, "total_per_gpu": 105.65507450828882, "tpot_ms": 10.234012605354284, "tpot_relative": 1.0, "ttft_ms": 341.22944840128184, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"w4a8_mxfp4_mxfp8\", distribution: \"power_law_1.01\", topk: 6, num_experts: 384, hidden_size: 7168, inter_size: 3072, moe_tp_size: 4, moe_ep_size: 1 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30431262611", "measured": {"e2e_ms": 12793.677557987121, "interactivity_tok_s": 76.99105557917564, "output_per_gpu": 141.4700583235486, "total_per_gpu": 629.2207879686715, "tpot_ms": 12.988521750706814, "tpot_relative": 1.269152, "ttft_ms": 781.0091190626736, "ttft_relative": 2.288809, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"w4a8_mxfp4_mxfp8\", distribution: \"power_law_1.01\", topk: 6, num_experts: 384, hidden_size: 7168, inter_size: 3072, moe_tp_size: 4, moe_ep_size: 1 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30431262611", "measured": {"e2e_ms": 14629.970523381553, "interactivity_tok_s": 66.23326872626104, "output_per_gpu": 241.93041652485363, "total_per_gpu": 1091.6807273301872, "tpot_ms": 15.098152623766047, "tpot_relative": 1.475292, "ttft_ms": 828.313946975095, "ttft_relative": 2.42744, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"w4a8_mxfp4_mxfp8\", distribution: \"power_law_1.01\", topk: 6, num_experts: 384, hidden_size: 7168, inter_size: 3072, moe_tp_size: 4, moe_ep_size: 1 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30431262611", "measured": {"e2e_ms": 17823.857376665637, "interactivity_tok_s": 56.36291276236397, "output_per_gpu": 399.6039612892553, "total_per_gpu": 1786.6929292706775, "tpot_ms": 17.74216325930807, "tpot_relative": 1.733647, "ttft_ms": 1424.7376164251136, "ttft_relative": 4.175307, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.21, "tpot_shape_error_pct": 15.69, "ttft_mape_pct": 36.31, "ttft_shape_error_pct": 117.52}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "42e62b61f1691637", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12056.052, "interactivity_tok_s": 98.62905611993294, "output_per_gpu": 5.6525, "total_per_gpu": null, "tpot_error_pct": 7.37, "tpot_ms": 10.139, "tpot_relative": 0.926298, "ttft_error_pct": 66.15, "ttft_ms": 1683.855, "ttft_relative": 0.338507, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=288400343040 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 15146.161374804797, "interactivity_tok_s": 91.35986267346672, "output_per_gpu": 7.677383823479054, "total_per_gpu": 34.366257611645516, "tpot_ms": 10.945725734879263, "tpot_relative": 1.0, "ttft_ms": 4974.363520101178, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 24060.957, "interactivity_tok_s": 45.71637560574198, "output_per_gpu": 84.198375, "total_per_gpu": null, "tpot_error_pct": 7.41, "tpot_ms": 21.874, "tpot_relative": 1.998406, "ttft_error_pct": 18.34, "ttft_ms": 1683.855, "ttft_relative": 0.338507, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=288400343040 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 20884.235502658976, "interactivity_tok_s": 49.10499830664149, "output_per_gpu": 172.04481326551291, "total_per_gpu": 769.3327025356712, "tpot_ms": 20.364525699713734, "tpot_relative": 1.8605, "ttft_ms": 2062.0297960126663, "ttft_relative": 0.414531, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 28940.667, "interactivity_tok_s": 37.53190211679928, "output_per_gpu": 138.25025, "total_per_gpu": null, "tpot_error_pct": 7.02, "tpot_ms": 26.644, "tpot_relative": 2.434192, "ttft_error_pct": 24.14, "ttft_ms": 1683.855, "ttft_relative": 0.338507, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=288400343040 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 25139.634143389958, "interactivity_tok_s": 40.166624273179636, "output_per_gpu": 285.18029604788467, "total_per_gpu": 1285.4306290401287, "tpot_ms": 24.89629183669606, "tpot_relative": 2.274522, "ttft_ms": 2219.707148993484, "ttft_relative": 0.446229, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 34504.764, "interactivity_tok_s": 31.169155004207838, "output_per_gpu": 229.632, "total_per_gpu": null, "tpot_error_pct": 7.05, "tpot_ms": 32.083, "tpot_relative": 2.931098, "ttft_error_pct": 36.62, "ttft_ms": 1683.855, "ttft_relative": 0.338507, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=288400343040 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 30199.25737457072, "interactivity_tok_s": 33.36688191230138, "output_per_gpu": 471.7453667588774, "total_per_gpu": 2129.7342443342427, "tpot_ms": 29.969836637067655, "tpot_relative": 2.73804, "ttft_ms": 2656.9001577385507, "ttft_relative": 0.534119, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.63, "tpot_shape_error_pct": 30.93, "ttft_mape_pct": 69.39, "ttft_shape_error_pct": 94.41}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.59, "tpot_shape_error_pct": 1.65, "ttft_mape_pct": 30.96, "ttft_shape_error_pct": 15.65}, "framework": "dynamo-vllm", "id": "6f9ce9766cd3654e", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 33890.964, "interactivity_tok_s": 31.76317377632373, "output_per_gpu": 468.037125, "total_per_gpu": null, "tpot_error_pct": 5.4, "tpot_ms": 31.483, "tpot_relative": 0.946041, "ttft_error_pct": 42.01, "ttft_ms": 1683.855, "ttft_relative": 0.579877, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30698.747163944347, "interactivity_tok_s": 34.660667921107205, "output_per_gpu": 460.69897681320793, "total_per_gpu": 4145.098669231182, "tpot_error_pct": 13.3, "tpot_ms": 28.85114626977609, "tpot_relative": 0.866956, "ttft_error_pct": 42.08, "ttft_ms": 4125.619019765205, "ttft_relative": 1.420758, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 33554.731484632975, "interactivity_tok_s": 30.0492626810796, "output_per_gpu": 842.8084517325678, "total_per_gpu": 3791.86838531026, "tpot_ms": 33.27868675558705, "tpot_relative": 1.0, "ttft_ms": 2903.8158593933986, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 50285.562, "interactivity_tok_s": 21.048643414931906, "output_per_gpu": 492.5951875, "total_per_gpu": null, "tpot_error_pct": 23.87, "tpot_ms": 47.509, "tpot_relative": 1.42761, "ttft_error_pct": 96.76, "ttft_ms": 1683.855, "ttft_relative": 0.579877, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 93412.78063895096, "interactivity_tok_s": 29.585311144495755, "output_per_gpu": 602.4798821847681, "total_per_gpu": 5421.206904264754, "tpot_error_pct": 11.87, "tpot_ms": 33.80055714526587, "tpot_relative": 1.015682, "ttft_error_pct": 19.84, "ttft_ms": 62288.24583863337, "ttft_relative": 21.450481, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 87290.00383508814, "interactivity_tok_s": 26.07235379293092, "output_per_gpu": 1291.5272723177704, "total_per_gpu": 5811.381219786419, "tpot_ms": 38.354803250296996, "tpot_relative": 1.152534, "ttft_ms": 51974.34864206298, "ttft_relative": 17.898638, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 106.53, "tpot_shape_error_pct": null, "ttft_mape_pct": 98.73, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.98, "tpot_shape_error_pct": null, "ttft_mape_pct": 12.52, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "76ee4f5ee9568350", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 90644.958, "interactivity_tok_s": 11.499407780499304, "output_per_gpu": 656.7935833333333, "total_per_gpu": null, "tpot_error_pct": 106.53, "tpot_ms": 86.961, "tpot_relative": 2.06535, "ttft_error_pct": 98.73, "ttft_ms": 1683.855, "ttft_relative": 0.012746, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 183918.0606571197, "interactivity_tok_s": 26.093978153133502, "output_per_gpu": 813.9847941705722, "total_per_gpu": 7327.470122705065, "tpot_error_pct": 8.98, "tpot_ms": 38.323018212534016, "tpot_relative": 0.910183, "ttft_error_pct": 12.52, "ttft_ms": 148645.64301945822, "ttft_relative": 1.125212, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28380189316", "measured": {"e2e_ms": 170857.90609730946, "interactivity_tok_s": 23.750301069659027, "output_per_gpu": 2620.6757123145508, "total_per_gpu": 7864.139969689436, "tpot_ms": 42.10472941235673, "tpot_relative": 1.0, "ttft_ms": 132104.55455120114, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.04, "tpot_shape_error_pct": 17.06, "ttft_mape_pct": 76.06, "ttft_shape_error_pct": 41.42}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.58, "tpot_shape_error_pct": 3.22, "ttft_mape_pct": 14.43, "ttft_shape_error_pct": 3.55}, "framework": "vllm", "id": "9efe122aeaddafc9", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 87535.32891391213, "interactivity_tok_s": 11.867331994091874, "output_per_gpu": 747.9494372425261, "total_per_gpu": null, "tpot_error_pct": 37.38, "tpot_ms": 84.26493844596645, "tpot_relative": 0.626211, "ttft_error_pct": 69.81, "ttft_ms": 1332.296883688459, "ttft_relative": 0.301941, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 125647.201307008, "interactivity_tok_s": 7.565807120009063, "output_per_gpu": 464.79453291324097, "total_per_gpu": 4181.948076315695, "tpot_error_pct": 1.78, "tpot_ms": 132.17360476390286, "tpot_relative": 0.982242, "ttft_error_pct": 15.92, "ttft_ms": 3709.790321837114, "ttft_relative": 0.840757, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24920699270", "measured": {"e2e_ms": 128555.74741127703, "interactivity_tok_s": 7.431453870848171, "output_per_gpu": 453.6675869055136, "total_per_gpu": 4081.8343547522577, "tpot_ms": 134.5631712689172, "tpot_relative": 1.0, "ttft_ms": 4412.439898153252, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 139158.7124353387, "interactivity_tok_s": 7.427896107006955, "output_per_gpu": 940.9687522140898, "total_per_gpu": null, "tpot_error_pct": 26.7, "tpot_ms": 134.62762343386444, "tpot_relative": 1.000479, "ttft_error_pct": 82.31, "ttft_ms": 1434.653662495384, "ttft_relative": 0.325138, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 178656.29550939557, "interactivity_tok_s": 5.370408016834428, "output_per_gpu": 654.4336851815741, "total_per_gpu": 5889.950710640664, "tpot_error_pct": 1.39, "tpot_ms": 186.20559124471276, "tpot_relative": 1.383778, "ttft_error_pct": 12.94, "ttft_ms": 7062.1483692816355, "ttft_relative": 1.600509, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24920699270", "measured": {"e2e_ms": 177348.67716652562, "interactivity_tok_s": 5.444959662815838, "output_per_gpu": 657.774125535181, "total_per_gpu": 5920.014916503052, "tpot_ms": 183.6560896546393, "tpot_relative": 1.364832, "ttft_ms": 8111.582449937647, "ttft_relative": 1.838344, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.74, "tpot_shape_error_pct": 19.88, "ttft_mape_pct": 25.33, "ttft_shape_error_pct": 25.87}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.7, "tpot_shape_error_pct": 15.9, "ttft_mape_pct": 6.1, "ttft_shape_error_pct": 7.63}, "framework": "vllm", "id": "7ce6c10354b4be91", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10765.63468144847, "interactivity_tok_s": 98.7804353720062, "output_per_gpu": 23.756146996211275, "total_per_gpu": null, "tpot_error_pct": 5.43, "tpot_ms": 10.123462163676535, "tpot_relative": 0.945652, "ttft_error_pct": 29.35, "ttft_ms": 409.33288800737307, "ttft_relative": 1.293474, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9736.129488200017, "interactivity_tok_s": 98.69624290283733, "output_per_gpu": 23.88782937633235, "total_per_gpu": 213.83240665843843, "tpot_error_pct": 5.35, "tpot_ms": 10.132097946063274, "tpot_relative": 0.946459, "ttft_error_pct": 1.22, "ttft_ms": 320.30977370000136, "ttft_relative": 1.012165, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 10264.800757798366, "interactivity_tok_s": 93.41193744382275, "output_per_gpu": 22.65688348521817, "total_per_gpu": 202.81356864613872, "tpot_ms": 10.705269876255297, "tpot_relative": 1.0, "ttft_ms": 316.46008130628616, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12149.495101131455, "interactivity_tok_s": 87.55854424912314, "output_per_gpu": 42.10051493846564, "total_per_gpu": null, "tpot_error_pct": 1.77, "tpot_ms": 11.420929945509167, "tpot_relative": 1.066851, "ttft_error_pct": 35.68, "ttft_ms": 465.88376687557655, "ttft_relative": 1.472172, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10641.419737150005, "interactivity_tok_s": 90.82238549364405, "output_per_gpu": 43.81692132907923, "total_per_gpu": 390.4730713404377, "tpot_error_pct": 5.3, "tpot_ms": 11.010501371052209, "tpot_relative": 1.028512, "ttft_error_pct": 2.41, "ttft_ms": 351.6394050999999, "ttft_relative": 1.111165, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 11207.270991746918, "interactivity_tok_s": 86.0062309877395, "output_per_gpu": 41.59724268921613, "total_per_gpu": 370.6924772319031, "tpot_ms": 11.627064557015103, "tpot_relative": 1.086107, "ttft_ms": 343.3604955469491, "ttft_relative": 1.085004, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14284.103914659856, "interactivity_tok_s": 74.31180832893553, "output_per_gpu": 71.61807321704579, "total_per_gpu": null, "tpot_error_pct": 3.27, "tpot_ms": 13.456811541627093, "tpot_relative": 1.257027, "ttft_error_pct": 36.46, "ttft_ms": 517.7857075753396, "ttft_relative": 1.63618, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12308.518275149967, "interactivity_tok_s": 77.04647475395745, "output_per_gpu": 73.08515345303125, "total_per_gpu": 656.9919425540729, "tpot_error_pct": 0.4, "tpot_ms": 12.979179166774733, "tpot_relative": 1.21241, "ttft_error_pct": 0.01, "ttft_ms": 379.4802182749998, "ttft_relative": 1.199141, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 12354.837196724839, "interactivity_tok_s": 76.74175594904166, "output_per_gpu": 72.71291403806806, "total_per_gpu": 653.6457322120638, "tpot_ms": 13.030715646694658, "tpot_relative": 1.217224, "ttft_ms": 379.44193705916405, "ttft_relative": 1.19902, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17902.437230052677, "interactivity_tok_s": 59.02151776640876, "output_per_gpu": 114.28611499697908, "total_per_gpu": null, "tpot_error_pct": 6.27, "tpot_ms": 16.94297330607, "tpot_relative": 1.582676, "ttft_error_pct": 4.8, "ttft_ms": 569.7755379430674, "ttft_relative": 1.800466, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15570.852375462477, "interactivity_tok_s": 61.238228207545006, "output_per_gpu": 117.15767800643685, "total_per_gpu": 1042.1717124508916, "tpot_error_pct": 2.43, "tpot_ms": 16.32966905265219, "tpot_relative": 1.525386, "ttft_error_pct": 25.45, "ttft_ms": 446.1753011499989, "ttft_relative": 1.409894, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 15382.612752582645, "interactivity_tok_s": 62.724201957589294, "output_per_gpu": 118.46879751216078, "total_per_gpu": 1053.8347266362105, "tpot_ms": 15.942809454572986, "tpot_relative": 1.489249, "ttft_ms": 598.4874058456626, "ttft_relative": 1.891194, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24985.62183468492, "interactivity_tok_s": 41.989045804503434, "output_per_gpu": 163.77419089564165, "total_per_gpu": null, "tpot_error_pct": 22.62, "tpot_ms": 23.81573529095885, "tpot_relative": 2.224674, "ttft_error_pct": 2.93, "ttft_ms": 622.1246320340165, "ttft_relative": 1.965887, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21669.294848362475, "interactivity_tok_s": 43.405847270926145, "output_per_gpu": 164.89166628587938, "total_per_gpu": 1488.1060163277398, "tpot_error_pct": 18.61, "tpot_ms": 23.038370700571814, "tpot_relative": 2.152059, "ttft_error_pct": 3.25, "ttft_ms": 584.7688576999993, "ttft_relative": 1.847844, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 18380.39630065614, "interactivity_tok_s": 51.48523885020374, "output_per_gpu": 193.43247365663942, "total_per_gpu": 1745.679659168149, "tpot_ms": 19.42304284359055, "tpot_relative": 1.814344, "ttft_ms": 604.4271546794334, "ttft_relative": 1.909963, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35891.27845566446, "interactivity_tok_s": 29.049032664020565, "output_per_gpu": 228.0219694628453, "total_per_gpu": null, "tpot_error_pct": 27.79, "tpot_ms": 34.424554220649696, "tpot_relative": 3.215664, "ttft_error_pct": 21.61, "ttft_ms": 674.9594879398221, "ttft_relative": 2.132842, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31724.06806723726, "interactivity_tok_s": 29.953766970755375, "output_per_gpu": 228.88405982672143, "total_per_gpu": 2046.7541412541334, "tpot_error_pct": 23.93, "tpot_ms": 33.38478265442625, "tpot_relative": 3.118537, "ttft_error_pct": 5.17, "ttft_ms": 816.4754740406255, "ttft_relative": 2.580027, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 25805.8626583952, "interactivity_tok_s": 37.12097929701371, "output_per_gpu": 280.13414262190645, "total_per_gpu": 2505.048699993063, "tpot_ms": 26.938944471231864, "tpot_relative": 2.516419, "ttft_ms": 860.974380155676, "ttft_relative": 2.720641, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 52034.8315817133, "interactivity_tok_s": 19.939133825161633, "output_per_gpu": 314.5585274797399, "total_per_gpu": null, "tpot_error_pct": 22.0, "tpot_ms": 50.15262993711782, "tpot_relative": 4.684854, "ttft_error_pct": 46.46, "ttft_ms": 728.6911560417624, "ttft_relative": 2.302632, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46379.012972598255, "interactivity_tok_s": 20.456109286878938, "output_per_gpu": 313.40108046806984, "total_per_gpu": 2824.927756752323, "tpot_error_pct": 18.91, "tpot_ms": 48.88515142229051, "tpot_relative": 4.566457, "ttft_error_pct": 5.21, "ttft_ms": 1290.084639326562, "ttft_relative": 4.076611, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 39285.32200956106, "interactivity_tok_s": 24.324766417045648, "output_per_gpu": 369.6315521570529, "total_per_gpu": 3331.776744038019, "tpot_ms": 41.11036393341262, "tpot_relative": 3.840199, "ttft_ms": 1360.9968583083173, "ttft_relative": 4.30069, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.69, "tpot_shape_error_pct": 4.48, "ttft_mape_pct": 19.86, "ttft_shape_error_pct": 42.31}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.85, "tpot_shape_error_pct": 1.91, "ttft_mape_pct": 17.99, "ttft_shape_error_pct": 35.41}, "framework": "vllm", "id": "055358a4ad761623", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9975.771771244961, "interactivity_tok_s": 106.52609208456332, "output_per_gpu": 12.818557093356738, "total_per_gpu": null, "tpot_error_pct": 7.45, "tpot_ms": 9.38737149210517, "tpot_relative": 0.925508, "ttft_error_pct": 13.56, "ttft_ms": 372.4907348213709, "ttft_relative": 0.864443, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9019.010758100012, "interactivity_tok_s": 106.41594215423277, "output_per_gpu": 12.89359810282538, "total_per_gpu": 115.41731437287824, "tpot_error_pct": 7.35, "tpot_ms": 9.397088253474852, "tpot_relative": 0.926466, "ttft_error_pct": 33.56, "ttft_ms": 286.27035899999913, "ttft_relative": 0.664351, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 9856.646435300354, "interactivity_tok_s": 98.59073063140173, "output_per_gpu": 11.797113934361768, "total_per_gpu": 105.60211329656138, "tpot_ms": 10.14294136574229, "tpot_relative": 1.0, "ttft_ms": 430.90256929863244, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10920.727334756446, "interactivity_tok_s": 97.45450437506031, "output_per_gpu": 23.41876984567198, "total_per_gpu": null, "tpot_error_pct": 5.11, "tpot_ms": 10.261198355197948, "tpot_relative": 1.011659, "ttft_error_pct": 7.23, "ttft_ms": 423.52141738894505, "ttft_relative": 0.98287, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9575.479501700025, "interactivity_tok_s": 100.91839859109169, "output_per_gpu": 24.34514960795334, "total_per_gpu": 216.9510100507957, "tpot_error_pct": 8.37, "tpot_ms": 9.908995921069565, "tpot_relative": 0.976935, "ttft_error_pct": 20.24, "ttft_ms": 315.0123952499995, "ttft_relative": 0.731052, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 10492.457879794529, "interactivity_tok_s": 92.47131755404325, "output_per_gpu": 22.208735479861907, "total_per_gpu": 197.91242493465393, "tpot_ms": 10.81416407217911, "tpot_relative": 1.066176, "ttft_ms": 394.9550791498041, "ttft_relative": 0.916576, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12841.701493570841, "interactivity_tok_s": 82.69319864296871, "output_per_gpu": 39.83117036757792, "total_per_gpu": null, "tpot_error_pct": 1.5, "tpot_ms": 12.092892963513737, "tpot_relative": 1.192247, "ttft_error_pct": 38.8, "ttft_ms": 470.67199189628815, "ttft_relative": 1.092293, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11077.295178375116, "interactivity_tok_s": 85.59499822582238, "output_per_gpu": 40.594606381786065, "total_per_gpu": 364.92130130269874, "tpot_error_pct": 4.84, "tpot_ms": 11.682925646680124, "tpot_relative": 1.151828, "ttft_error_pct": 0.16, "ttft_ms": 339.663433949999, "ttft_relative": 0.78826, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 11625.030297473131, "interactivity_tok_s": 81.45242792421789, "output_per_gpu": 38.64745539341294, "total_per_gpu": 347.4175751715158, "tpot_ms": 12.277104875626113, "tpot_relative": 1.210409, "ttft_ms": 339.10630592581583, "ttft_relative": 0.786967, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 41.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.6, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.51, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "f1d30d6732bb7684", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 270429.34471461916, "interactivity_tok_s": 3.809115735118987, "output_per_gpu": 968.4156143497196, "total_per_gpu": null, "tpot_error_pct": 41.32, "tpot_ms": 262.52812188936093, "tpot_relative": 1.413195, "ttft_error_pct": 93.6, "ttft_ms": 1863.076021802914, "ttft_relative": 0.063986, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 273247.8384348906, "interactivity_tok_s": 3.9409657923503914, "output_per_gpu": 856.7524367585485, "total_per_gpu": 7703.773850483689, "tpot_error_pct": 36.59, "tpot_ms": 253.74490738819637, "tpot_relative": 1.365915, "ttft_error_pct": 35.51, "ttft_ms": 39456.96160187428, "ttft_relative": 1.355131, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 200278.60158102788, "interactivity_tok_s": 5.383024154535669, "output_per_gpu": 1166.3575092094495, "total_per_gpu": 10487.690602618402, "tpot_ms": 185.76918313796017, "tpot_relative": 1.0, "ttft_ms": 29116.71926343836, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.81, "tpot_shape_error_pct": 15.93, "ttft_mape_pct": 69.45, "ttft_shape_error_pct": 40.71}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.74, "tpot_shape_error_pct": 2.22, "ttft_mape_pct": 9.39, "ttft_shape_error_pct": 4.82}, "framework": "vllm", "id": "07463659008a16e5", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 87535.32891391213, "interactivity_tok_s": 11.867331994091874, "output_per_gpu": 747.9494372425261, "total_per_gpu": null, "tpot_error_pct": 23.87, "tpot_ms": 84.26493844596645, "tpot_relative": 0.761303, "ttft_error_pct": 61.64, "ttft_ms": 1332.296883688459, "ttft_relative": 0.383627, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 125647.201307008, "interactivity_tok_s": 7.565807120009063, "output_per_gpu": 464.79453291324097, "total_per_gpu": 4181.948076315695, "tpot_error_pct": 19.41, "tpot_ms": 132.17360476390286, "tpot_relative": 1.19414, "ttft_error_pct": 6.82, "ttft_ms": 3709.790321837114, "ttft_relative": 1.068212, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 105579.83804140438, "interactivity_tok_s": 9.034631397840283, "output_per_gpu": 552.4683578466035, "total_per_gpu": 4970.785632612325, "tpot_ms": 110.68520185992854, "tpot_relative": 1.0, "ttft_ms": 3472.89853157763, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 139158.7124353387, "interactivity_tok_s": 7.427896107006955, "output_per_gpu": 940.9687522140898, "total_per_gpu": null, "tpot_error_pct": 11.74, "tpot_ms": 134.62762343386444, "tpot_relative": 1.216311, "ttft_error_pct": 77.25, "ttft_ms": 1434.653662495384, "ttft_relative": 0.4131, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 178656.29550939557, "interactivity_tok_s": 5.370408016834428, "output_per_gpu": 654.4336851815741, "total_per_gpu": 5889.950710640664, "tpot_error_pct": 22.07, "tpot_ms": 186.20559124471276, "tpot_relative": 1.682299, "ttft_error_pct": 11.97, "ttft_ms": 7062.1483692816355, "ttft_relative": 2.033503, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29177379558", "measured": {"e2e_ms": 146868.67898824322, "interactivity_tok_s": 6.555600716644086, "output_per_gpu": 794.7767200078339, "total_per_gpu": 7153.048219869655, "tpot_ms": 152.54132202729934, "tpot_relative": 1.378155, "ttft_ms": 6307.328692249132, "ttft_relative": 1.816157, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "af0cecdb6d64a602", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 11710.86570136249, "interactivity_tok_s": 92.60643234614732, "output_per_gpu": 9.929402701277354, "total_per_gpu": 44.4469651391264, "tpot_ms": 10.798385972393016, "tpot_relative": 1.0, "ttft_ms": 1671.7472126707435, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cf3b97e2fb4a7e0a", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 48240.095717965574, "interactivity_tok_s": 20.496373151649188, "output_per_gpu": 587.7854167508495, "total_per_gpu": 2116.044581131448, "tpot_ms": 48.78911954818395, "tpot_relative": 1.0, "ttft_ms": 3301.2894753210644, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 55569.920459040375, "interactivity_tok_s": 18.590655455301754, "output_per_gpu": 1016.5206617027675, "total_per_gpu": 3658.7238794212744, "tpot_ms": 53.79046491417903, "tpot_relative": 1.102509, "ttft_ms": 6039.767584700144, "ttft_relative": 1.829518, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e77bdf1dc8204507", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 158936.26300967194, "interactivity_tok_s": 24.82102147989814, "output_per_gpu": 2823.677133294063, "total_per_gpu": 8473.307895780827, "tpot_ms": 40.28843054706159, "tpot_relative": 1.0, "ttft_ms": 121854.03752837922, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "adf79d0bc2f791f8", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 91498.0493381261, "interactivity_tok_s": 13.205884850432728, "output_per_gpu": 2434.424723723641, "total_per_gpu": 4869.917460869118, "tpot_ms": 75.72381641410664, "tpot_relative": 1.0, "ttft_ms": 21804.288176852962, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (188157660536 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24921817337", "measured": {"e2e_ms": 310309.1468536016, "interactivity_tok_s": 6.030685661868361, "output_per_gpu": 1476.8170521883494, "total_per_gpu": 2953.9831388369043, "tpot_ms": 165.81862429390674, "tpot_relative": 2.189782, "ttft_ms": 157642.89885259123, "ttft_relative": 7.229903, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b22229b13fac1e0b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 27888.11571631377, "interactivity_tok_s": 43.75500469738771, "output_per_gpu": 255.37692292203982, "total_per_gpu": 1838.340711226123, "tpot_ms": 22.854528457168755, "tpot_relative": 1.0, "ttft_ms": 6839.265504325158, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "fe669b8a4b840e3c", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 107202.83133797397, "interactivity_tok_s": 22.59777377378904, "output_per_gpu": 4173.643906168134, "total_per_gpu": 9393.222435205302, "tpot_ms": 44.25214669419743, "tpot_relative": 1.0, "ttft_ms": 66473.05924285641, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f81ca004453b1bcd", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 31618.614715421063, "interactivity_tok_s": 32.754401973869946, "output_per_gpu": 901.5620313034717, "total_per_gpu": 4056.2058399723937, "tpot_ms": 30.53024753124044, "tpot_relative": 1.0, "ttft_ms": 3500.6653990119958, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (243858017656 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30118489668", "measured": {"e2e_ms": 41971.8377690062, "interactivity_tok_s": 30.009777027442066, "output_per_gpu": 1338.399591335268, "total_per_gpu": 6023.230219512599, "tpot_ms": 33.322473508735584, "tpot_relative": 1.091458, "ttft_ms": 11280.791635487549, "ttft_relative": 3.222471, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 12.5, "points": 1, "status_counts": {"failed": 7, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.29, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.02, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "18a119db9a90ad2f", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"w4a8_mxfp4_mxfp8\", distribution: \"power_law_1.01\", topk: 6, num_experts: 384, hidden_size: 7168, inter_size: 3072, moe_tp_size: 4, moe_ep_size: 1 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "31024844958", "measured": {"e2e_ms": 10562.154804519378, "interactivity_tok_s": 95.52875890651308, "output_per_gpu": 22.01836981386817, "total_per_gpu": 98.54895004942951, "tpot_ms": 10.468051835349666, "tpot_relative": 1.0, "ttft_ms": 834.9183805985376, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "c44c91d56a970b32", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (284869922168 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 79521.4852826604, "interactivity_tok_s": 19.098614207159073, "output_per_gpu": 5623.82847589358, "total_per_gpu": 12657.014589443015, "tpot_ms": 52.359819888144145, "tpot_relative": 1.0, "ttft_ms": 31330.71093956719, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "8c1d62d0f48d507a", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (284869922168 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 3072, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 48298.04997252009, "interactivity_tok_s": 37.291434196494414, "output_per_gpu": 1587.8064396860075, "total_per_gpu": 7622.185493962466, "tpot_ms": 26.815809623487343, "tpot_relative": 1.0, "ttft_ms": 23606.782007329744, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 60}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "fd256216b2fc44ec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: DSV4 module data missing for ModuleKey { fmha_quant: \"bfloat16\", kv_quant: \"bfloat16\", gemm_quant: \"fp8_block\" }", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 23844.685812798343, "interactivity_tok_s": 45.49640292599674, "output_per_gpu": 287.58384336291857, "total_per_gpu": 2221.664369890705, "tpot_ms": 21.97975962246013, "tpot_relative": 1.0, "ttft_ms": 3625.8456106130325, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.29, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.02, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "8b0c3d1b206e7363", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10248.67525375559, "interactivity_tok_s": 96.99113948163985, "output_per_gpu": 21.621517656700185, "total_per_gpu": 194.1438219944571, "tpot_error_pct": 24.29, "tpot_ms": 10.310220143246148, "tpot_relative": 0.757097, "ttft_error_pct": 53.02, "ttft_ms": 763.2042398722147, "ttft_relative": 0.469825, "unavailable_metrics": {}}, "concurrency": 18, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25203314223", "measured": {"e2e_ms": 14155.106936355716, "interactivity_tok_s": 73.4316694418399, "output_per_gpu": 16.369654705429372, "total_per_gpu": 138.83720069016942, "tpot_ms": 13.618102483588913, "tpot_relative": 1.0, "ttft_ms": 1624.4449138165994, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "a77419138d80b6a8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: DSV4 module data missing for ModuleKey { fmha_quant: \"bfloat16\", kv_quant: \"bfloat16\", gemm_quant: \"fp8_block\" }", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 18, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 11498.782565352869, "interactivity_tok_s": 86.59948632715843, "output_per_gpu": 38.55144249585685, "total_per_gpu": 311.58249884110455, "tpot_ms": 11.547412604991289, "tpot_relative": 1.0, "ttft_ms": 874.7955290781748, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "6b823cb78ba0d729", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (284869922168 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 116235.28097925433, "interactivity_tok_s": 24.818108115081895, "output_per_gpu": 3843.4166454215447, "total_per_gpu": 11533.348563273683, "tpot_ms": 40.29315995252284, "tpot_relative": 1.0, "ttft_ms": 79149.56274805825, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e7444efdcd2fbaeb", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (284869922168 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28728578034", "measured": {"e2e_ms": 94038.48715200914, "interactivity_tok_s": 21.99389714439495, "output_per_gpu": 4710.586771770225, "total_per_gpu": 12116.192630940455, "tpot_ms": 45.46715815913714, "tpot_relative": 1.0, "ttft_ms": 52190.867714513195, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 79}]}, {"aic": {"coverage_pct": 61.61, "points": 138, "status_counts": {"failed": 86, "success": 138, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.24, "tpot_shape_error_pct": 7.88, "ttft_mape_pct": 43.36, "ttft_shape_error_pct": 37.33}, "aisimulate": {"coverage_pct": 100.0, "points": 224, "status_counts": {"failed": 0, "success": 224, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.12, "tpot_shape_error_pct": 10.66, "ttft_mape_pct": 35.86, "ttft_shape_error_pct": 21.54}, "frameworks": ["dynamo-vllm", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "hf_model_paths": ["MiniMaxAI/MiniMax-M2.5"], "model": "MiniMaxAI/MiniMax-M2.5", "precisions": ["fp8"], "rows": 224, "workloads": [{"aic": {"coverage_pct": 64.41, "points": 76, "status_counts": {"failed": 42, "success": 76, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.14, "tpot_shape_error_pct": 7.7, "ttft_mape_pct": 47.36, "ttft_shape_error_pct": 33.82}, "aisimulate": {"coverage_pct": 100.0, "points": 118, "status_counts": {"failed": 0, "success": 118, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.46, "tpot_shape_error_pct": 9.43, "ttft_mape_pct": 43.48, "ttft_shape_error_pct": 29.65}, "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 47.37, "points": 9, "status_counts": {"failed": 10, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.51, "tpot_shape_error_pct": 4.27, "ttft_mape_pct": 65.25, "ttft_shape_error_pct": 33.16}, "aisimulate": {"coverage_pct": 100.0, "points": 19, "status_counts": {"failed": 0, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.34, "tpot_shape_error_pct": 7.98, "ttft_mape_pct": 34.68, "ttft_shape_error_pct": 11.81}, "gpu": "b200", "precisions": ["fp8"], "rows": 19, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.75, "tpot_shape_error_pct": 48.21, "ttft_mape_pct": 30.74, "ttft_shape_error_pct": 0.37}, "framework": "dynamo-vllm", "id": "d9e8ccdd95c1bcde", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 37899.66545695322, "interactivity_tok_s": 25.64583698602305, "output_per_gpu": 6024.93861196862, "total_per_gpu": 12054.255501317934, "tpot_error_pct": 4.8, "tpot_ms": 38.99268331717927, "tpot_relative": 0.952039, "ttft_error_pct": 30.62, "ttft_ms": 2011.635635048718, "ttft_relative": 0.693837, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 40595.08414662612, "interactivity_tok_s": 24.415835883200508, "output_per_gpu": 7485.688811465882, "total_per_gpu": 11253.532156701805, "tpot_ms": 40.95702497279879, "tpot_relative": 1.0, "ttft_ms": 2899.2896960721964, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 63216.308653214524, "interactivity_tok_s": 19.06463600556234, "output_per_gpu": 7245.240610773666, "total_per_gpu": 14491.999730046678, "tpot_error_pct": 50.69, "tpot_ms": 52.45313887494301, "tpot_relative": 1.280687, "ttft_error_pct": 30.87, "ttft_ms": 14929.665125131778, "ttft_relative": 5.149422, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 119533.64105199065, "interactivity_tok_s": 9.399851618159659, "output_per_gpu": 5134.975051807568, "total_per_gpu": 7718.036524144188, "tpot_ms": 106.38465803737698, "tpot_relative": 2.59747, "ttft_ms": 21596.6283822112, "ttft_relative": 7.448938, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.81, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "0c9cb14cdadac805", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 63493.62858228793, "interactivity_tok_s": 21.926924277901815, "output_per_gpu": 5664.039544862025, "total_per_gpu": 11332.195104478684, "tpot_error_pct": 24.5, "tpot_ms": 45.60603153118974, "tpot_relative": 0.755027, "ttft_error_pct": 49.81, "ttft_ms": 21535.79022769388, "ttft_relative": 0.501878, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 98502.92083795118, "interactivity_tok_s": 16.555412959160666, "output_per_gpu": 4496.094799372965, "total_per_gpu": 7209.767676771704, "tpot_ms": 60.403204828947885, "tpot_relative": 1.0, "ttft_ms": 42910.370189530724, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.15, "tpot_shape_error_pct": null, "ttft_mape_pct": 37.8, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b6c2bcb49d67a12b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20652.446607849626, "interactivity_tok_s": 46.86098505380227, "output_per_gpu": 3139.0965144938305, "total_per_gpu": 6278.0081537440265, "tpot_error_pct": 8.15, "tpot_ms": 21.33971359867649, "tpot_relative": 0.918494, "ttft_error_pct": 37.8, "ttft_ms": 1000.6300126609397, "ttft_relative": 0.62201, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 23000.426418037296, "interactivity_tok_s": 43.041543492417745, "output_per_gpu": 3293.174340806018, "total_per_gpu": 5656.513283616664, "tpot_ms": 23.23336755281932, "tpot_relative": 1.0, "ttft_ms": 1608.703264914584, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.4, "tpot_shape_error_pct": 5.94, "ttft_mape_pct": 65.38, "ttft_shape_error_pct": 19.34}, "framework": "dynamo-vllm", "id": "b0b6bc803b944e44", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10931.383623746882, "interactivity_tok_s": 85.22704907886113, "output_per_gpu": 437.67978480548464, "total_per_gpu": 873.9750739945625, "tpot_error_pct": 4.17, "tpot_ms": 11.733364123339454, "tpot_relative": 0.958273, "ttft_error_pct": 68.52, "ttft_ms": 105.57442100625057, "ttft_relative": 0.314767, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 11644.894751755055, "interactivity_tok_s": 81.67074487894246, "output_per_gpu": 620.5377041138911, "total_per_gpu": 826.9781623530647, "tpot_ms": 12.244286512658398, "tpot_relative": 1.0, "ttft_ms": 335.4050148584065, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13657.255777453134, "interactivity_tok_s": 68.11698155010889, "output_per_gpu": 697.2737982194012, "total_per_gpu": 1394.8619813393743, "tpot_error_pct": 3.01, "tpot_ms": 14.680627021976452, "tpot_relative": 1.198978, "ttft_error_pct": 69.9, "ttft_ms": 139.91343153906234, "ttft_relative": 0.417148, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 13581.317293138272, "interactivity_tok_s": 70.16791001606775, "output_per_gpu": 1056.028147513961, "total_per_gpu": 1411.5639269498888, "tpot_ms": 14.251528936390011, "tpot_relative": 1.163933, "ttft_ms": 464.76901224959875, "ttft_relative": 1.385695, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15585.453782048411, "interactivity_tok_s": 59.7216788942164, "output_per_gpu": 1216.7359722746057, "total_per_gpu": 2436.1449177891072, "tpot_error_pct": 0.03, "tpot_ms": 16.744338379556883, "tpot_relative": 1.367523, "ttft_error_pct": 57.72, "ttft_ms": 198.89135273124967, "ttft_relative": 0.592989, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 15853.359191590607, "interactivity_tok_s": 59.737432598501194, "output_per_gpu": 1804.3005441147682, "total_per_gpu": 2413.9372008328023, "tpot_ms": 16.739922633117814, "tpot_relative": 1.367162, "ttft_ms": 470.4406847591599, "ttft_relative": 1.402605, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 68.42, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "83c47656b93241f0", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17229.52154498484, "interactivity_tok_s": 54.07160656780479, "output_per_gpu": 1648.788851367391, "total_per_gpu": 3296.0588729872106, "tpot_error_pct": 11.59, "tpot_ms": 18.493994602250602, "tpot_relative": 0.884087, "ttft_error_pct": 68.42, "ttft_ms": 190.9979016007798, "ttft_relative": 0.315813, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 19870.612113974676, "interactivity_tok_s": 47.80401129746323, "output_per_gpu": 2865.602482948973, "total_per_gpu": 2870.761917327119, "tpot_ms": 20.918746625203525, "tpot_relative": 1.0, "ttft_ms": 604.7807827039833, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.46, "tpot_shape_error_pct": 4.43, "ttft_mape_pct": 53.03, "ttft_shape_error_pct": 34.57}, "framework": "dynamo-vllm", "id": "baddb2755529593c", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17379.453407430447, "interactivity_tok_s": 53.961692311912174, "output_per_gpu": 2178.001853527571, "total_per_gpu": 4353.997377377051, "tpot_error_pct": 11.42, "tpot_ms": 18.531664911836867, "tpot_relative": 0.885769, "ttft_error_pct": 59.95, "ttft_ms": 306.6440822564474, "ttft_relative": 0.400462, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 20034.15032637986, "interactivity_tok_s": 47.79759883704417, "output_per_gpu": 2845.7587617212143, "total_per_gpu": 3801.1766240710713, "tpot_ms": 20.921553055610367, "tpot_relative": 1.0, "ttft_ms": 765.7249326322926, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22875.2873352338, "interactivity_tok_s": 41.188106844937906, "output_per_gpu": 3322.047118951845, "total_per_gpu": 6643.898587891769, "tpot_error_pct": 7.5, "tpot_ms": 24.27885320791096, "tpot_relative": 1.160471, "ttft_error_pct": 46.11, "ttft_ms": 516.9065493086928, "ttft_relative": 0.675055, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 25125.027148190973, "interactivity_tok_s": 38.09930991669618, "output_per_gpu": 4529.64097365174, "total_per_gpu": 6051.365703767552, "tpot_ms": 26.24719456038683, "tpot_relative": 1.254553, "ttft_ms": 959.1819971711175, "ttft_relative": 1.252646, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.99, "tpot_shape_error_pct": null, "ttft_mape_pct": 88.5, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.54, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.18, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "aaa3a7e7cd4914b4", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 50157.96176356557, "interactivity_tok_s": 20.482527139034318, "output_per_gpu": 5221.264796095319, "total_per_gpu": null, "tpot_error_pct": 4.99, "tpot_ms": 48.822100574409234, "tpot_relative": 1.049888, "ttft_error_pct": 88.5, "ttft_ms": 212.95287594492635, "ttft_relative": 0.115009, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 53737.011286743844, "interactivity_tok_s": 17.548207408729994, "output_per_gpu": 4306.09727396045, "total_per_gpu": 8608.22786132534, "tpot_error_pct": 22.54, "tpot_ms": 56.98587762887471, "tpot_relative": 1.225445, "ttft_error_pct": 35.18, "ttft_ms": 1200.2684682570336, "ttft_relative": 0.648228, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 44724.15922835098, "interactivity_tok_s": 21.50436220782919, "output_per_gpu": 5104.413896221695, "total_per_gpu": 10204.125713300225, "tpot_ms": 46.50219291953358, "tpot_relative": 1.0, "ttft_ms": 1851.6138814925398, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.18, "tpot_shape_error_pct": 4.71, "ttft_mape_pct": 61.35, "ttft_shape_error_pct": 30.16}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.96, "tpot_shape_error_pct": 2.99, "ttft_mape_pct": 15.04, "ttft_shape_error_pct": 7.93}, "framework": "vllm", "id": "967a9cad1973a913", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8662.440828522318, "interactivity_tok_s": 119.56501725916998, "output_per_gpu": 118.09604478123845, "total_per_gpu": null, "tpot_error_pct": 17.81, "tpot_ms": 8.363650362985295, "tpot_relative": 1.178089, "ttft_error_pct": 107.69, "ttft_ms": 106.4265071883623, "ttft_relative": 2.076881, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7683.334645049508, "interactivity_tok_s": 119.96720905802671, "output_per_gpu": 115.89130577979256, "total_per_gpu": 232.91598426859235, "tpot_error_pct": 17.41, "tpot_ms": 8.33561110450033, "tpot_relative": 1.174139, "ttft_error_pct": 19.31, "ttft_ms": 41.346573599999985, "ttft_relative": 0.806866, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 6559.775767032988, "interactivity_tok_s": 140.8582331365337, "output_per_gpu": 135.62584351901475, "total_per_gpu": 272.57805598905856, "tpot_ms": 7.099336529592143, "tpot_relative": 1.0, "ttft_ms": 51.24343553325161, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9686.341239685595, "interactivity_tok_s": 106.90473124189404, "output_per_gpu": 211.2252654921344, "total_per_gpu": null, "tpot_error_pct": 17.6, "tpot_ms": 9.354122950248978, "tpot_relative": 1.317605, "ttft_error_pct": 97.82, "ttft_ms": 117.07346158089007, "ttft_relative": 2.284653, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8731.089185399973, "interactivity_tok_s": 106.89619845993678, "output_per_gpu": 208.20291744348089, "total_per_gpu": 414.8679692453764, "tpot_error_pct": 17.61, "tpot_ms": 9.354869624992194, "tpot_relative": 1.31771, "ttft_error_pct": 19.86, "ttft_ms": 47.42660471250019, "ttft_relative": 0.925516, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 7442.68520087935, "interactivity_tok_s": 125.71617802759263, "output_per_gpu": 243.71024044243248, "total_per_gpu": 485.62034470098723, "tpot_ms": 7.9544257206142275, "tpot_relative": 1.120446, "ttft_ms": 59.182358434190974, "ttft_relative": 1.154926, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10922.37652125736, "interactivity_tok_s": 94.7692708250876, "output_per_gpu": 374.6437409510708, "total_per_gpu": null, "tpot_error_pct": 12.74, "tpot_ms": 10.551943591986328, "tpot_relative": 1.486328, "ttft_error_pct": 84.05, "ttft_ms": 127.73822665534661, "ttft_relative": 2.492772, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9767.90961719995, "interactivity_tok_s": 94.23289698412441, "output_per_gpu": 366.4253162386902, "total_per_gpu": 736.7513544042712, "tpot_error_pct": 13.38, "tpot_ms": 10.612005276336477, "tpot_relative": 1.494788, "ttft_error_pct": 17.47, "ttft_ms": 57.27855589374981, "ttft_relative": 1.117774, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 8633.272904035402, "interactivity_tok_s": 106.84505978616782, "output_per_gpu": 413.8777364961901, "total_per_gpu": 832.1613420474293, "tpot_ms": 9.359347095704093, "tpot_relative": 1.318341, "ttft_ms": 69.40586218843237, "ttft_relative": 1.354434, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12530.957613599854, "interactivity_tok_s": 82.54944328139335, "output_per_gpu": 653.1025203626817, "total_per_gpu": null, "tpot_error_pct": 4.6, "tpot_ms": 12.113952078286154, "tpot_relative": 1.70635, "ttft_error_pct": 51.84, "ttft_ms": 138.38463751311855, "ttft_relative": 2.700534, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11390.692149734361, "interactivity_tok_s": 81.57119824675472, "output_per_gpu": 631.1888051993024, "total_per_gpu": 1260.380995146402, "tpot_error_pct": 5.86, "tpot_ms": 12.259229010893986, "tpot_relative": 1.726813, "ttft_error_pct": 16.52, "ttft_ms": 76.07941174062462, "ttft_relative": 1.484666, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 10777.755035375594, "interactivity_tok_s": 86.34793507591647, "output_per_gpu": 666.8292445995929, "total_per_gpu": 1331.5491339168752, "tpot_ms": 11.581052854602804, "tpot_relative": 1.631287, "ttft_ms": 91.13675738335587, "ttft_relative": 1.778506, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16167.126315468464, "interactivity_tok_s": 63.865721389247526, "output_per_gpu": 1012.4248231016383, "total_per_gpu": null, "tpot_error_pct": 15.3, "tpot_ms": 15.657851790402866, "tpot_relative": 2.205537, "ttft_error_pct": 9.14, "ttft_ms": 149.14393388633235, "ttft_relative": 2.910498, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14959.736316863942, "interactivity_tok_s": 62.07126958516195, "output_per_gpu": 958.1964590044477, "total_per_gpu": 1916.8249470041937, "tpot_error_pct": 18.63, "tpot_ms": 16.11051307123656, "tpot_relative": 2.269298, "ttft_error_pct": 14.42, "ttft_ms": 116.93928228906259, "ttft_relative": 2.282034, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 12645.133019235527, "interactivity_tok_s": 73.63516297314455, "output_per_gpu": 1132.059202140475, "total_per_gpu": 2264.6288240337294, "tpot_ms": 13.580468347231195, "tpot_relative": 1.912921, "ttft_ms": 136.64760816391208, "ttft_relative": 2.666636, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19878.562682812353, "interactivity_tok_s": 51.88184586258011, "output_per_gpu": 1646.7991434966575, "total_per_gpu": null, "tpot_error_pct": 11.05, "tpot_ms": 19.274564799577654, "tpot_relative": 2.714981, "ttft_error_pct": 17.56, "ttft_ms": 160.68289284441303, "ttft_relative": 3.135678, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18848.926697564068, "interactivity_tok_s": 49.30939164351746, "output_per_gpu": 1519.218631494689, "total_per_gpu": 3041.7747419824123, "tpot_error_pct": 16.85, "tpot_ms": 20.28011230050263, "tpot_relative": 2.856621, "ttft_error_pct": 2.62, "ttft_ms": 200.02474394921884, "ttft_relative": 3.903422, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 16148.932948800848, "interactivity_tok_s": 57.61705625770033, "output_per_gpu": 1772.6036436876893, "total_per_gpu": 3549.101412487552, "tpot_ms": 17.35597173738555, "tpot_relative": 2.444732, "ttft_ms": 194.91545380742537, "ttft_relative": 3.803716, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.26, "tpot_shape_error_pct": 2.05, "ttft_mape_pct": 65.31, "ttft_shape_error_pct": 48.21}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.15, "tpot_shape_error_pct": 0.28, "ttft_mape_pct": 6.87, "ttft_shape_error_pct": 4.79}, "framework": "vllm", "id": "4be51484c255a70c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 24441.90247423516, "interactivity_tok_s": 42.14601507338685, "output_per_gpu": 2678.6785549535566, "total_per_gpu": null, "tpot_error_pct": 6.35, "tpot_ms": 23.72703559894684, "tpot_relative": 1.063481, "ttft_error_pct": 54.29, "ttft_ms": 169.14505651254072, "ttft_relative": 0.457107, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24451.784501185262, "interactivity_tok_s": 38.31241225388381, "output_per_gpu": 2352.6045357833164, "total_per_gpu": 4702.935286913584, "tpot_error_pct": 16.99, "tpot_ms": 26.101201703858464, "tpot_relative": 1.169895, "ttft_error_pct": 4.37, "ttft_ms": 386.21212714726516, "ttft_relative": 1.043722, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 20931.925860177398, "interactivity_tok_s": 44.82148197441413, "output_per_gpu": 2744.208076825911, "total_per_gpu": 5485.76388544654, "tpot_ms": 22.310730389745693, "tpot_relative": 1.0, "ttft_ms": 370.03352788487973, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33419.76759543163, "interactivity_tok_s": 30.77609676423655, "output_per_gpu": 3918.1601016848367, "total_per_gpu": null, "tpot_error_pct": 4.17, "tpot_ms": 32.492749410706715, "tpot_relative": 1.456373, "ttft_error_pct": 76.32, "ttft_ms": 179.68494827865797, "ttft_relative": 0.485591, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34571.45290148117, "interactivity_tok_s": 27.326271772528262, "output_per_gpu": 3340.082280616911, "total_per_gpu": 6677.08774741201, "tpot_error_pct": 17.32, "tpot_ms": 36.594820117588206, "tpot_relative": 1.640234, "ttft_error_pct": 9.37, "ttft_ms": 830.0599889150383, "ttft_relative": 2.243202, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 29511.101382326295, "interactivity_tok_s": 32.05893988506565, "output_per_gpu": 3904.482763361824, "total_per_gpu": 7805.368799001383, "tpot_ms": 31.192547338904383, "tpot_relative": 1.398096, "ttft_ms": 758.9502933461063, "ttft_relative": 2.051031, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 90.91, "points": 30, "status_counts": {"failed": 3, "success": 30, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.41, "tpot_shape_error_pct": 6.35, "ttft_mape_pct": 69.54, "ttft_shape_error_pct": 46.54}, "aisimulate": {"coverage_pct": 100.0, "points": 33, "status_counts": {"failed": 0, "success": 33, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.52, "tpot_shape_error_pct": 10.63, "ttft_mape_pct": 40.03, "ttft_shape_error_pct": 41.21}, "gpu": "b300", "precisions": ["fp8"], "rows": 33, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 34.98, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "312c8c428d65e3d1", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 33499.921027152646, "interactivity_tok_s": 28.128488963576125, "output_per_gpu": 3414.1114253194914, "total_per_gpu": 6828.021778553866, "tpot_error_pct": 8.97, "tpot_ms": 35.551145363510656, "tpot_relative": 1.089654, "ttft_error_pct": 34.98, "ttft_ms": 761.2645353451178, "ttft_relative": 0.650241, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 31209.862263177503, "interactivity_tok_s": 30.650334086698965, "output_per_gpu": 4877.363217542652, "total_per_gpu": 7330.392685652113, "tpot_ms": 32.626071780208115, "tpot_relative": 1.0, "ttft_ms": 1170.7430066046788, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.96, "tpot_shape_error_pct": 23.24, "ttft_mape_pct": 79.35, "ttft_shape_error_pct": 50.47}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.18, "tpot_shape_error_pct": 20.03, "ttft_mape_pct": 41.04, "ttft_shape_error_pct": 3.16}, "framework": "dynamo-vllm", "id": "d9f3a8d12576e94b", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 17312.326, "interactivity_tok_s": 59.59120433823968, "output_per_gpu": 1404.8297, "total_per_gpu": null, "tpot_error_pct": 5.2, "tpot_ms": 16.781, "tpot_relative": 0.948047, "ttft_error_pct": 68.88, "ttft_ms": 145.363, "ttft_relative": 0.311183, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15125.575640314471, "interactivity_tok_s": 62.034151659726845, "output_per_gpu": 1499.9784065191475, "total_per_gpu": 2998.507088773646, "tpot_error_pct": 8.93, "tpot_ms": 16.12015274239995, "tpot_relative": 0.910712, "ttft_error_pct": 40.23, "ttft_ms": 279.2124032476571, "ttft_relative": 0.597719, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 16768.97523010457, "interactivity_tok_s": 56.495269636881126, "output_per_gpu": 1683.0524661900351, "total_per_gpu": 2696.911940272122, "tpot_ms": 17.700596995596637, "tpot_relative": 1.0, "ttft_ms": 467.129788496095, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22405.843, "interactivity_tok_s": 45.955882352941174, "output_per_gpu": 2166.8782, "total_per_gpu": null, "tpot_error_pct": 10.99, "tpot_ms": 21.76, "tpot_relative": 1.229337, "ttft_error_pct": 80.12, "ttft_ms": 145.363, "ttft_relative": 0.311183, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19346.555867440355, "interactivity_tok_s": 48.73112237989248, "output_per_gpu": 2355.0315234894474, "total_per_gpu": 4707.893641277627, "tpot_error_pct": 4.67, "tpot_ms": 20.520766835705423, "tpot_relative": 1.159326, "ttft_error_pct": 39.56, "ttft_ms": 441.9690073388656, "ttft_relative": 0.946137, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 18787.162614940975, "interactivity_tok_s": 51.008146251458776, "output_per_gpu": 3029.6782417657187, "total_per_gpu": 4856.2129436823125, "tpot_ms": 19.604711668411223, "tpot_relative": 1.107573, "ttft_ms": 731.2361397033101, "ttft_relative": 1.565381, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30398.542, "interactivity_tok_s": 33.81462820816285, "output_per_gpu": 2282.4346, "total_per_gpu": null, "tpot_error_pct": 22.69, "tpot_ms": 29.573, "tpot_relative": 1.670735, "ttft_error_pct": 89.05, "ttft_ms": 145.363, "ttft_relative": 0.311183, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26046.79085190506, "interactivity_tok_s": 36.40700558152777, "output_per_gpu": 3506.821708913001, "total_per_gpu": 7013.436885623241, "tpot_error_pct": 13.95, "tpot_ms": 27.467241098987312, "tpot_relative": 1.551769, "ttft_error_pct": 43.34, "ttft_ms": 752.4884480072284, "ttft_relative": 1.610877, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 23521.953584809842, "interactivity_tok_s": 41.48556538869053, "output_per_gpu": 4833.820936003463, "total_per_gpu": 7749.281252205636, "tpot_ms": 24.10476971039696, "tpot_relative": 1.361805, "ttft_ms": 1328.0766616885985, "ttft_relative": 2.843057, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.77, "tpot_shape_error_pct": 2.79, "ttft_mape_pct": 88.05, "ttft_shape_error_pct": 63.57}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.94, "tpot_shape_error_pct": 14.18, "ttft_mape_pct": 106.97, "ttft_shape_error_pct": 219.7}, "framework": "dynamo-vllm", "id": "4ed1501f2e9ff09d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 19668.295, "interactivity_tok_s": 52.39991616013415, "output_per_gpu": 1268.0192222222222, "total_per_gpu": null, "tpot_error_pct": 16.02, "tpot_ms": 19.084, "tpot_relative": 1.160226, "ttft_error_pct": 79.25, "ttft_ms": 145.363, "ttft_relative": 0.207458, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16630.3911091407, "interactivity_tok_s": 56.96204629708196, "output_per_gpu": 1522.813744860668, "total_per_gpu": 3044.1556951762477, "tpot_error_pct": 6.73, "tpot_ms": 17.555549089380726, "tpot_relative": 1.067303, "ttft_error_pct": 33.78, "ttft_ms": 464.02661888476615, "ttft_relative": 0.662245, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 15848.986347244681, "interactivity_tok_s": 60.79575240904546, "output_per_gpu": 1797.0907426885544, "total_per_gpu": 3199.606785270607, "tpot_ms": 16.448517542340927, "tpot_relative": 1.0, "ttft_ms": 700.6869354733226, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24068.218, "interactivity_tok_s": 42.762454564892025, "output_per_gpu": 1268.0192222222222, "total_per_gpu": null, "tpot_error_pct": 10.91, "tpot_ms": 23.385, "tpot_relative": 1.421709, "ttft_error_pct": 87.57, "ttft_ms": 145.363, "ttft_relative": 0.207458, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20249.026628575935, "interactivity_tok_s": 47.357451699636634, "output_per_gpu": 2501.0148088949136, "total_per_gpu": 4999.72573534441, "tpot_error_pct": 0.15, "tpot_ms": 21.116001053909596, "tpot_relative": 1.283763, "ttft_error_pct": 31.86, "ttft_ms": 796.7278866283211, "ttft_relative": 1.137067, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 20588.236032409623, "interactivity_tok_s": 47.42749904081311, "output_per_gpu": 2770.846288657389, "total_per_gpu": 4934.817986219905, "tpot_ms": 21.08481408938437, "tpot_relative": 1.281867, "ttft_ms": 1169.1660794212567, "ttft_relative": 1.6686, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30836.386, "interactivity_tok_s": 33.332222259258025, "output_per_gpu": 1268.0192222222222, "total_per_gpu": null, "tpot_error_pct": 17.38, "tpot_ms": 30.001, "tpot_relative": 1.823933, "ttft_error_pct": 97.32, "ttft_ms": 145.363, "ttft_relative": 0.207458, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 38798.79979490554, "interactivity_tok_s": 47.11110167348696, "output_per_gpu": 2590.9975334424294, "total_per_gpu": 5181.842471608466, "tpot_error_pct": 16.95, "tpot_ms": 21.226419346563, "tpot_relative": 1.290476, "ttft_error_pct": 255.29, "ttft_ms": 19250.991179898534, "ttft_relative": 27.474454, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 28951.249842257606, "interactivity_tok_s": 39.12527114039532, "output_per_gpu": 3936.4971638338375, "total_per_gpu": 7011.9417318846145, "tpot_ms": 25.55892830522876, "tpot_relative": 1.553874, "ttft_ms": 5418.336622430593, "ttft_relative": 7.732892, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.45, "tpot_shape_error_pct": 3.83, "ttft_mape_pct": 40.35, "ttft_shape_error_pct": 48.53}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.43, "tpot_shape_error_pct": 3.85, "ttft_mape_pct": 43.62, "ttft_shape_error_pct": 15.52}, "framework": "dynamo-vllm", "id": "13868ff504160945", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10104.268, "interactivity_tok_s": 102.72213662044172, "output_per_gpu": 151.298, "total_per_gpu": null, "tpot_error_pct": 18.24, "tpot_ms": 9.735, "tpot_relative": 1.182385, "ttft_error_pct": 2.5, "ttft_ms": 145.363, "ttft_relative": 0.975035, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8469.930019912468, "interactivity_tok_s": 110.83569054889901, "output_per_gpu": 171.61956280062887, "total_per_gpu": 341.9714784794222, "tpot_error_pct": 9.58, "tpot_ms": 9.022364502333437, "tpot_relative": 1.09583, "ttft_error_pct": 35.63, "ttft_ms": 95.96699494999925, "ttft_relative": 0.643707, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 7767.928375938573, "interactivity_tok_s": 121.45710485048522, "output_per_gpu": 230.25738424115522, "total_per_gpu": 368.64297630992155, "tpot_ms": 8.233359433612458, "tpot_relative": 1.0, "ttft_ms": 149.08496705029393, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11117.038, "interactivity_tok_s": 93.24009324009324, "output_per_gpu": 274.7106, "total_per_gpu": null, "tpot_error_pct": 16.18, "tpot_ms": 10.725, "tpot_relative": 1.302627, "ttft_error_pct": 20.44, "ttft_ms": 145.363, "ttft_relative": 0.975035, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9238.551477468827, "interactivity_tok_s": 100.23416924948076, "output_per_gpu": 309.42650510975307, "total_per_gpu": 622.1469604455232, "tpot_error_pct": 8.07, "tpot_ms": 9.97663778218205, "tpot_relative": 1.211734, "ttft_error_pct": 39.49, "ttft_ms": 110.56096070000244, "ttft_relative": 0.741597, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 8621.308213675275, "interactivity_tok_s": 108.32514361698186, "output_per_gpu": 414.99401864811506, "total_per_gpu": 668.663608440492, "tpot_ms": 9.231467105511712, "tpot_relative": 1.121227, "ttft_ms": 182.70426551243872, "ttft_relative": 1.225504, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12731.332, "interactivity_tok_s": 81.28098837681866, "output_per_gpu": 479.1418, "total_per_gpu": null, "tpot_error_pct": 5.54, "tpot_ms": 12.303, "tpot_relative": 1.494287, "ttft_error_pct": 46.82, "ttft_ms": 145.363, "ttft_relative": 0.975035, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10603.302189984392, "interactivity_tok_s": 88.17428568241003, "output_per_gpu": 541.5652760040063, "total_per_gpu": 1081.4174394159859, "tpot_error_pct": 2.71, "tpot_ms": 11.3411749498243, "tpot_relative": 1.377466, "ttft_error_pct": 49.02, "ttft_ms": 139.33115858124967, "ttft_relative": 0.934576, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 11040.885950418624, "interactivity_tok_s": 85.78098417435592, "output_per_gpu": 657.0085086746159, "total_per_gpu": 1050.6984872352066, "tpot_ms": 11.657595324010613, "tpot_relative": 1.415898, "ttft_ms": 273.3264599807626, "ttft_relative": 1.83336, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16348.66, "interactivity_tok_s": 63.13529894564051, "output_per_gpu": 744.2432, "total_per_gpu": null, "tpot_error_pct": 16.98, "tpot_ms": 15.839, "tpot_relative": 1.923759, "ttft_error_pct": 58.64, "ttft_ms": 145.363, "ttft_relative": 0.975035, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13514.704432834353, "interactivity_tok_s": 69.11135528856227, "output_per_gpu": 846.1902437466263, "total_per_gpu": 1692.762015432958, "tpot_error_pct": 6.86, "tpot_ms": 14.469402254154568, "tpot_relative": 1.757412, "ttft_error_pct": 45.39, "ttft_ms": 191.90070874062496, "ttft_relative": 1.28719, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 12813.67628383598, "interactivity_tok_s": 73.85492800152825, "output_per_gpu": 1120.5753661468393, "total_per_gpu": 1797.4109133419256, "tpot_ms": 13.54005788184246, "tpot_relative": 1.644536, "ttft_ms": 351.4186918498581, "ttft_relative": 2.357171, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19672.387, "interactivity_tok_s": 52.38893545683151, "output_per_gpu": 1235.0668, "total_per_gpu": null, "tpot_error_pct": 20.33, "tpot_ms": 19.088, "tpot_relative": 2.318373, "ttft_error_pct": 73.36, "ttft_ms": 145.363, "ttft_relative": 0.975035, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16303.953773353194, "interactivity_tok_s": 57.34677852077186, "output_per_gpu": 1398.484160366734, "total_per_gpu": 2800.040565511511, "tpot_error_pct": 9.93, "tpot_ms": 17.437771149390457, "tpot_relative": 2.117941, "ttft_error_pct": 48.58, "ttft_ms": 280.6257754296891, "ttft_relative": 1.882321, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 15122.950114408524, "interactivity_tok_s": 63.03912895105012, "output_per_gpu": 1896.002461238383, "total_per_gpu": 3043.948008982762, "tpot_ms": 15.863163350123381, "tpot_relative": 1.926694, "ttft_ms": 545.7333300516666, "ttft_relative": 3.660552, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 60.0, "points": 3, "status_counts": {"failed": 2, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.47, "tpot_shape_error_pct": 13.04, "ttft_mape_pct": 87.59, "ttft_shape_error_pct": 56.94}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.77, "tpot_shape_error_pct": 29.14, "ttft_mape_pct": 39.42, "ttft_shape_error_pct": 54.97}, "framework": "dynamo-vllm", "id": "3a50065378e722b2", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 23189.461, "interactivity_tok_s": 44.39314569830418, "output_per_gpu": 2093.1133, "total_per_gpu": null, "tpot_error_pct": 8.08, "tpot_ms": 22.526, "tpot_relative": 1.080759, "ttft_error_pct": 79.99, "ttft_ms": 145.363, "ttft_relative": 0.200067, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19748.438913834005, "interactivity_tok_s": 47.71973926164496, "output_per_gpu": 2307.0201369390393, "total_per_gpu": 4611.915095260257, "tpot_error_pct": 0.54, "tpot_ms": 20.955688682979797, "tpot_relative": 1.005418, "ttft_error_pct": 39.08, "ttft_ms": 442.6089023798833, "ttft_relative": 0.609175, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 19922.595722766782, "interactivity_tok_s": 47.97828823993695, "output_per_gpu": 2864.4257925297275, "total_per_gpu": 4591.332973297299, "tpot_ms": 20.84276110475329, "tpot_relative": 1.0, "ttft_ms": 726.570698330157, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29510.578, "interactivity_tok_s": 34.83713638738896, "output_per_gpu": 2282.4346, "total_per_gpu": null, "tpot_error_pct": 17.07, "tpot_ms": 28.705, "tpot_relative": 1.377217, "ttft_error_pct": 88.22, "ttft_ms": 145.363, "ttft_relative": 0.200067, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25034.86486158603, "interactivity_tok_s": 37.93155920387906, "output_per_gpu": 3640.4460032438706, "total_per_gpu": 7280.6776045607285, "tpot_error_pct": 7.52, "tpot_ms": 26.363271665820037, "tpot_relative": 1.264865, "ttft_error_pct": 38.65, "ttft_ms": 757.1338224183544, "ttft_relative": 1.042065, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 23809.661655308733, "interactivity_tok_s": 40.78444797801413, "output_per_gpu": 4766.466976031971, "total_per_gpu": 7641.303570330553, "tpot_ms": 24.519150057862127, "tpot_relative": 1.176387, "ttft_ms": 1234.2055949560056, "ttft_relative": 1.698672, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 43407.01, "interactivity_tok_s": 23.64681122750597, "output_per_gpu": 2282.4346, "total_per_gpu": null, "tpot_error_pct": 27.27, "tpot_ms": 42.289, "tpot_relative": 2.028954, "ttft_error_pct": 94.55, "ttft_ms": 145.363, "ttft_relative": 0.200067, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35964.13127670005, "interactivity_tok_s": 26.716475262049716, "output_per_gpu": 5082.795471390802, "total_per_gpu": 10163.362356300726, "tpot_error_pct": 12.64, "tpot_ms": 37.43008724734293, "tpot_relative": 1.795832, "ttft_error_pct": 44.56, "ttft_ms": 1478.1971586722611, "ttft_relative": 2.034485, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 33282.516090960235, "interactivity_tok_s": 30.09447319947776, "output_per_gpu": 6856.399394109311, "total_per_gpu": 10987.037291404991, "tpot_ms": 33.22869263640585, "tpot_relative": 1.594256, "ttft_ms": 2666.282283765658, "ttft_relative": 3.66968, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 68978.23303189143, "interactivity_tok_s": 26.556494069988464, "output_per_gpu": 5254.298998976693, "total_per_gpu": 10512.41625734825, "tpot_error_pct": 4.64, "tpot_ms": 37.65557295946311, "tpot_relative": 1.80665, "ttft_error_pct": 62.34, "ttft_ms": 34319.936947467286, "ttft_relative": 47.235509, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 57485.25723536962, "interactivity_tok_s": 25.32407483058008, "output_per_gpu": 7898.63643515557, "total_per_gpu": 12665.954834559516, "tpot_ms": 39.48811582219976, "tpot_relative": 1.894572, "ttft_ms": 21141.059267745237, "ttft_relative": 29.097044, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 136854.9046992884, "interactivity_tok_s": 12.820127969984505, "output_per_gpu": 5265.1205687271995, "total_per_gpu": 10531.344638464676, "tpot_error_pct": 93.49, "tpot_ms": 78.00234150090225, "tpot_relative": 3.742419, "ttft_error_pct": 12.47, "ttft_ms": 65255.12573914659, "ttft_relative": 89.812493, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 111658.13001013336, "interactivity_tok_s": 24.805265202298575, "output_per_gpu": 8079.7085019171245, "total_per_gpu": 12953.672349450335, "tpot_ms": 40.31402171452435, "tpot_relative": 1.934198, "ttft_ms": 74548.62528527375, "ttft_relative": 102.603402, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.66, "tpot_shape_error_pct": 4.13, "ttft_mape_pct": 69.54, "ttft_shape_error_pct": 50.16}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.83, "tpot_shape_error_pct": 4.92, "ttft_mape_pct": 42.31, "ttft_shape_error_pct": 23.14}, "framework": "dynamo-vllm", "id": "40430d51aab4f101", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11117.038, "interactivity_tok_s": 93.24009324009324, "output_per_gpu": 305.2338888888889, "total_per_gpu": null, "tpot_error_pct": 15.15, "tpot_ms": 10.725, "tpot_relative": 1.151513, "ttft_error_pct": 49.12, "ttft_ms": 145.363, "ttft_relative": 0.508792, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9445.592894500038, "interactivity_tok_s": 99.14849762809483, "output_per_gpu": 337.76206998579164, "total_per_gpu": 674.4557102165019, "tpot_error_pct": 8.29, "tpot_ms": 10.085881520373526, "tpot_relative": 1.082893, "ttft_error_pct": 51.32, "ttft_ms": 139.07087383125122, "ttft_relative": 0.486769, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 8888.92148510813, "interactivity_tok_s": 107.36721109077278, "output_per_gpu": 407.37842136761236, "total_per_gpu": 723.8734107563557, "tpot_ms": 9.313830450104154, "tpot_relative": 1.0, "ttft_ms": 285.70200618451054, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12731.332, "interactivity_tok_s": 81.28098837681866, "output_per_gpu": 532.3798888888889, "total_per_gpu": null, "tpot_error_pct": 4.73, "tpot_ms": 12.303, "tpot_relative": 1.320939, "ttft_error_pct": 59.59, "ttft_ms": 145.363, "ttft_relative": 0.508792, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10718.177375196881, "interactivity_tok_s": 87.52669045446993, "output_per_gpu": 589.3260501592536, "total_per_gpu": 1178.9178140340618, "tpot_error_pct": 2.75, "tpot_ms": 11.42508639144976, "tpot_relative": 1.22668, "ttft_error_pct": 44.95, "ttft_ms": 198.02579743750096, "ttft_relative": 0.69312, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 11172.592558680026, "interactivity_tok_s": 85.1223687794224, "output_per_gpu": 642.1568572082849, "total_per_gpu": 1144.4713315172214, "tpot_ms": 11.747793374868362, "tpot_relative": 1.261328, "ttft_ms": 359.73610184673817, "ttft_relative": 1.25913, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16348.66, "interactivity_tok_s": 63.13529894564051, "output_per_gpu": 826.9368888888889, "total_per_gpu": null, "tpot_error_pct": 16.18, "tpot_ms": 15.839, "tpot_relative": 1.700589, "ttft_error_pct": 69.4, "ttft_ms": 145.363, "ttft_relative": 0.508792, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13673.7011427, "interactivity_tok_s": 68.62673555213729, "output_per_gpu": 925.4710971717709, "total_per_gpu": 1852.9753054978148, "tpot_error_pct": 6.89, "tpot_ms": 14.571580477411421, "tpot_relative": 1.56451, "ttft_error_pct": 40.23, "ttft_ms": 283.964348445313, "ttft_relative": 0.993918, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 13002.475251627038, "interactivity_tok_s": 73.3527173104129, "output_per_gpu": 1101.032738743306, "total_per_gpu": 1964.0659061718059, "tpot_ms": 13.632760130319582, "tpot_relative": 1.463711, "ttft_ms": 475.0991618075659, "ttft_relative": 1.662919, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19672.387, "interactivity_tok_s": 52.38893545683151, "output_per_gpu": 1268.0192222222222, "total_per_gpu": null, "tpot_error_pct": 19.89, "tpot_ms": 19.088, "tpot_relative": 2.049425, "ttft_error_pct": 81.91, "ttft_ms": 145.363, "ttft_relative": 0.508792, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16599.72413639958, "interactivity_tok_s": 57.06484369991397, "output_per_gpu": 1526.454514159438, "total_per_gpu": 3051.4337149820694, "tpot_error_pct": 10.06, "tpot_ms": 17.523924279170636, "tpot_relative": 1.881495, "ttft_error_pct": 42.44, "ttft_ms": 462.54303959062565, "ttft_relative": 1.61897, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 15466.763418533674, "interactivity_tok_s": 62.80733279487253, "output_per_gpu": 1839.5420620192945, "total_per_gpu": 3275.188683361696, "tpot_ms": 15.92170779271872, "tpot_relative": 1.709469, "ttft_ms": 803.6104488480646, "ttft_relative": 2.812757, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25193.518, "interactivity_tok_s": 40.84133142740453, "output_per_gpu": 1268.0192222222222, "total_per_gpu": null, "tpot_error_pct": 12.33, "tpot_ms": 24.485, "tpot_relative": 2.628886, "ttft_error_pct": 87.65, "ttft_ms": 145.363, "ttft_relative": 0.508792, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21109.711721154512, "interactivity_tok_s": 45.34260559695905, "output_per_gpu": 2403.216154461491, "total_per_gpu": 4804.2185165491355, "tpot_error_pct": 1.18, "tpot_ms": 22.05431264556764, "tpot_relative": 2.36791, "ttft_error_pct": 32.61, "ttft_ms": 793.4211468175766, "ttft_relative": 2.777093, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 21251.84516908837, "interactivity_tok_s": 45.878028568913756, "output_per_gpu": 2695.6640729250635, "total_per_gpu": 4800.920067754166, "tpot_ms": 21.796926136394287, "tpot_relative": 2.340275, "ttft_ms": 1177.391067614593, "ttft_relative": 4.121046, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.91, "tpot_shape_error_pct": 0.2, "ttft_mape_pct": 84.22, "ttft_shape_error_pct": 65.6}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.68, "tpot_shape_error_pct": 8.82, "ttft_mape_pct": 32.99, "ttft_shape_error_pct": 10.04}, "framework": "vllm", "id": "8d8eaa2fdcf281a6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 49698.03044854273, "interactivity_tok_s": 20.67233883808677, "output_per_gpu": 5269.585084888997, "total_per_gpu": null, "tpot_error_pct": 11.8, "tpot_ms": 48.37382010000714, "tpot_relative": 1.117996, "ttft_error_pct": 76.52, "ttft_ms": 211.61248623543236, "ttft_relative": 0.234849, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52979.999511595175, "interactivity_tok_s": 17.784187066564098, "output_per_gpu": 4368.123055445531, "total_per_gpu": 8732.222287445044, "tpot_error_pct": 29.96, "tpot_ms": 56.22972791824101, "tpot_relative": 1.299558, "ttft_error_pct": 26.63, "ttft_ms": 1140.9932745699207, "ttft_relative": 1.26628, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 40775.863356442416, "interactivity_tok_s": 23.11159056609442, "output_per_gpu": 5650.389262103048, "total_per_gpu": 11295.573503994126, "tpot_ms": 43.268333139608224, "tpot_relative": 1.0, "ttft_ms": 901.0589709145279, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71710.91938929683, "interactivity_tok_s": 14.313299676940835, "output_per_gpu": 7303.992257533039, "total_per_gpu": null, "tpot_error_pct": 12.02, "tpot_ms": 69.86509208711885, "tpot_relative": 1.614693, "ttft_error_pct": 91.92, "ttft_ms": 238.9301841742377, "ttft_relative": 0.265166, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 85331.54375529251, "interactivity_tok_s": 11.338545381792986, "output_per_gpu": 5428.92300397654, "total_per_gpu": 10857.526274724618, "tpot_error_pct": 41.41, "tpot_ms": 88.19473453850286, "tpot_relative": 2.038321, "ttft_error_pct": 39.34, "ttft_ms": 4121.429583182017, "ttft_relative": 4.573984, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 60387.249129237185, "interactivity_tok_s": 16.034084188775747, "output_per_gpu": 7624.502885775462, "total_per_gpu": 15248.556730936185, "tpot_ms": 62.36714166063968, "tpot_relative": 1.441404, "ttft_ms": 2957.726833001817, "ttft_relative": 3.282501, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.33, "tpot_shape_error_pct": 4.52, "ttft_mape_pct": 64.31, "ttft_shape_error_pct": 25.47}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.18, "tpot_shape_error_pct": 2.49, "ttft_mape_pct": 13.88, "ttft_shape_error_pct": 22.24}, "framework": "vllm", "id": "465f1bec502e914e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8447.077269338133, "interactivity_tok_s": 122.63045560329714, "output_per_gpu": 121.10697787900746, "total_per_gpu": null, "tpot_error_pct": 16.22, "tpot_ms": 8.154581136312057, "tpot_relative": 1.162214, "ttft_error_pct": 89.19, "ttft_ms": 104.94076689089958, "ttft_relative": 1.891895, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7503.184952625024, "interactivity_tok_s": 122.84724836954886, "output_per_gpu": 118.62841011172429, "total_per_gpu": 238.41696076745984, "tpot_error_pct": 16.02, "tpot_ms": 8.140190466389624, "tpot_relative": 1.160163, "ttft_error_pct": 27.34, "ttft_ms": 40.30103975000081, "ttft_relative": 0.726556, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 6488.159513376013, "interactivity_tok_s": 142.5228410336775, "output_per_gpu": 137.1652139630403, "total_per_gpu": 275.6718513321317, "tpot_ms": 7.016419212157751, "tpot_relative": 1.0, "ttft_ms": 55.46861322654877, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9537.993324349338, "interactivity_tok_s": 108.56940812396422, "output_per_gpu": 214.51052967051368, "total_per_gpu": null, "tpot_error_pct": 17.7, "tpot_ms": 9.21069772120525, "tpot_relative": 1.312735, "ttft_error_pct": 116.11, "ttft_ms": 115.44955555636881, "ttft_relative": 2.081349, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8596.372783537478, "interactivity_tok_s": 108.5608821949982, "output_per_gpu": 211.4384528107378, "total_per_gpu": 421.3151410896416, "tpot_error_pct": 17.71, "tpot_ms": 9.211421091842174, "tpot_relative": 1.312838, "ttft_error_pct": 14.12, "ttft_ms": 45.87776328749957, "ttft_relative": 0.827094, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 7317.705012511578, "interactivity_tok_s": 127.78176970381872, "output_per_gpu": 248.5044369000022, "total_per_gpu": 495.1733258644488, "tpot_ms": 7.82584246812255, "tpot_relative": 1.115361, "ttft_ms": 53.421389919822104, "ttft_relative": 0.963092, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10669.308080306568, "interactivity_tok_s": 97.02780134997722, "output_per_gpu": 383.53002548993993, "total_per_gpu": null, "tpot_error_pct": 13.16, "tpot_ms": 10.306324435746216, "tpot_relative": 1.468887, "ttft_error_pct": 87.74, "ttft_ms": 125.9381825381888, "ttft_relative": 2.27044, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9549.54750440623, "interactivity_tok_s": 96.38115605087634, "output_per_gpu": 374.77276619146494, "total_per_gpu": 753.5351157493049, "tpot_error_pct": 13.92, "tpot_ms": 10.375472145946599, "tpot_relative": 1.478742, "ttft_error_pct": 17.42, "ttft_ms": 55.39732771249985, "ttft_relative": 0.998715, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 8400.675745808621, "interactivity_tok_s": 109.80030212408516, "output_per_gpu": 425.8340666795343, "total_per_gpu": 856.2012816092074, "tpot_ms": 9.107443063953516, "tpot_relative": 1.298019, "ttft_ms": 67.08038148717606, "ttft_relative": 1.209339, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12527.631254996111, "interactivity_tok_s": 82.55904289242719, "output_per_gpu": 653.2759332883589, "total_per_gpu": null, "tpot_error_pct": 5.09, "tpot_ms": 12.11254351995069, "tpot_relative": 1.726314, "ttft_error_pct": 53.51, "ttft_ms": 136.4992340865549, "ttft_relative": 2.460837, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11389.302956034382, "interactivity_tok_s": 81.56504189278473, "output_per_gpu": 631.478489008378, "total_per_gpu": 1260.9594464189104, "tpot_error_pct": 6.37, "tpot_ms": 12.26015431113829, "tpot_relative": 1.747352, "ttft_error_pct": 16.78, "ttft_ms": 73.99803108125055, "ttft_relative": 1.334052, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 10725.75240603528, "interactivity_tok_s": 86.76122901539192, "output_per_gpu": 670.8515002078564, "total_per_gpu": 1339.5809216870584, "tpot_ms": 11.525885598307909, "tpot_relative": 1.642702, "ttft_ms": 88.92016855061229, "ttft_relative": 1.603072, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15903.633121387731, "interactivity_tok_s": 64.92479952829306, "output_per_gpu": 1029.1987921921925, "total_per_gpu": null, "tpot_error_pct": 11.64, "tpot_ms": 15.40243492880125, "tpot_relative": 2.195199, "ttft_error_pct": 20.39, "ttft_ms": 146.94218922405187, "ttft_relative": 2.649105, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14733.45700405318, "interactivity_tok_s": 63.01664592988945, "output_per_gpu": 972.9171840147934, "total_per_gpu": 1946.2730342650632, "tpot_error_pct": 15.02, "tpot_ms": 15.868822994999954, "tpot_relative": 2.26167, "ttft_error_pct": 7.01, "ttft_ms": 113.49986324218783, "ttft_relative": 2.0462, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 12831.246303572698, "interactivity_tok_s": 72.48150374482479, "output_per_gpu": 1115.527386518857, "total_per_gpu": 2231.55773897074, "tpot_ms": 13.79662325329999, "tpot_relative": 1.966334, "ttft_ms": 122.05254026166585, "ttft_relative": 2.200389, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19568.803522907405, "interactivity_tok_s": 52.70387094651071, "output_per_gpu": 1672.866711635127, "total_per_gpu": null, "tpot_error_pct": 10.19, "tpot_ms": 18.97393838518811, "tpot_relative": 2.70422, "ttft_error_pct": 18.91, "ttft_ms": 158.46455485997063, "ttft_relative": 2.856833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18569.83103111729, "interactivity_tok_s": 50.041989621939365, "output_per_gpu": 1541.9759036453552, "total_per_gpu": 3087.3392803505417, "tpot_error_pct": 16.05, "tpot_ms": 19.983218244415703, "tpot_relative": 2.848065, "ttft_error_pct": 0.62, "ttft_ms": 194.20393387109374, "ttft_relative": 3.50115, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 16025.56142900794, "interactivity_tok_s": 58.07309857853164, "output_per_gpu": 1785.5962488947223, "total_per_gpu": 3575.1151655656186, "tpot_ms": 17.21967700152439, "tpot_relative": 2.454197, "ttft_ms": 195.41700380359543, "ttft_relative": 3.523019, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.9, "tpot_shape_error_pct": 0.58, "ttft_mape_pct": 62.76, "ttft_shape_error_pct": 47.58}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.23, "tpot_shape_error_pct": 3.15, "ttft_mape_pct": 13.25, "ttft_shape_error_pct": 6.34}, "framework": "vllm", "id": "dd014727d36e935f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 24163.339016524285, "interactivity_tok_s": 42.63162885003846, "output_per_gpu": 2709.559302016434, "total_per_gpu": null, "tpot_error_pct": 5.6, "tpot_ms": 23.456762665991775, "tpot_relative": 1.055974, "ttft_error_pct": 51.13, "ttft_ms": 167.07080921470026, "ttft_relative": 0.488702, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24213.623533372796, "interactivity_tok_s": 38.67637724283245, "output_per_gpu": 2375.717631069024, "total_per_gpu": 4749.139138753373, "tpot_error_pct": 16.4, "tpot_ms": 25.855575710243677, "tpot_relative": 1.163964, "ttft_error_pct": 9.77, "ttft_ms": 375.2644715386734, "ttft_relative": 1.097693, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 20812.476060266294, "interactivity_tok_s": 45.017912145339466, "output_per_gpu": 2758.7549784975886, "total_per_gpu": 5514.843629256613, "tpot_ms": 22.213380237882184, "tpot_relative": 1.0, "ttft_ms": 341.8663885609476, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32829.43688313271, "interactivity_tok_s": 31.33068989876696, "output_per_gpu": 3988.6154753777437, "total_per_gpu": null, "tpot_error_pct": 6.21, "tpot_ms": 31.917586342053568, "tpot_relative": 1.436863, "ttft_error_pct": 74.38, "ttft_ms": 177.74605521191037, "ttft_relative": 0.519928, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34076.22884778111, "interactivity_tok_s": 27.7162370241799, "output_per_gpu": 3387.774043303792, "total_per_gpu": 6772.427340133133, "tpot_error_pct": 20.06, "tpot_ms": 36.07993390760769, "tpot_relative": 1.624243, "ttft_error_pct": 16.72, "ttft_ms": 809.931046749414, "ttft_relative": 2.369145, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 28393.01085364473, "interactivity_tok_s": 33.27608374989837, "output_per_gpu": 4055.415161052622, "total_per_gpu": 8107.094558620278, "tpot_ms": 30.051613270238096, "tpot_relative": 1.352861, "ttft_ms": 693.8836138218107, "ttft_relative": 2.029692, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.92, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.86, "tpot_shape_error_pct": null, "ttft_mape_pct": 39.43, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "b982e33471672431", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 71710.91938929683, "interactivity_tok_s": 14.313299676940835, "output_per_gpu": 7303.992257533039, "total_per_gpu": null, "tpot_error_pct": 11.59, "tpot_ms": 69.86509208711885, "tpot_relative": 1.115885, "ttft_error_pct": 91.92, "ttft_ms": 238.9301841742377, "ttft_relative": 0.08083, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 85331.54375529251, "interactivity_tok_s": 11.338545381792986, "output_per_gpu": 5428.92300397654, "total_per_gpu": 10857.526274724618, "tpot_error_pct": 40.86, "tpot_ms": 88.19473453850286, "tpot_relative": 1.408646, "ttft_error_pct": 39.43, "ttft_ms": 4121.429583182017, "ttft_relative": 1.394272, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 60608.52530217899, "interactivity_tok_s": 15.971992974370806, "output_per_gpu": 7596.511050578217, "total_per_gpu": 15192.57470910437, "tpot_ms": 62.609594281980556, "tpot_relative": 1.0, "ttft_ms": 2955.9716842456055, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 33.33, "points": 6, "status_counts": {"failed": 12, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.81, "tpot_shape_error_pct": 5.61, "ttft_mape_pct": 44.83, "ttft_shape_error_pct": 36.95}, "aisimulate": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.0, "tpot_shape_error_pct": 8.97, "ttft_mape_pct": 55.57, "ttft_shape_error_pct": 26.98}, "gpu": "gb200", "precisions": ["fp8"], "rows": 18, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.64, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.49, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "045f701cbda75c64", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 58516.98621614977, "interactivity_tok_s": 24.90712112346337, "output_per_gpu": 6203.663319886282, "total_per_gpu": 12411.834795048528, "tpot_error_pct": 5.64, "tpot_ms": 40.14916035631133, "tpot_relative": 0.943607, "ttft_error_pct": 41.49, "ttft_ms": 21563.657188013858, "ttft_relative": 0.585107, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 76011.91376737578, "interactivity_tok_s": 23.50253779233909, "output_per_gpu": 5392.817339270239, "total_per_gpu": 8647.718047410159, "tpot_ms": 42.54859661691347, "tpot_relative": 1.0, "ttft_ms": 36854.2294594359, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 66.67, "points": 4, "status_counts": {"failed": 2, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.16, "tpot_shape_error_pct": 5.51, "ttft_mape_pct": 41.46, "ttft_shape_error_pct": 33.81}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.16, "tpot_shape_error_pct": 4.44, "ttft_mape_pct": 62.1, "ttft_shape_error_pct": 13.99}, "framework": "dynamo-vllm", "id": "f7e5f4611b953f92", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7644.794, "interactivity_tok_s": 135.556459265284, "output_per_gpu": 20.882833333333334, "total_per_gpu": null, "tpot_error_pct": 8.67, "tpot_ms": 7.377, "tpot_relative": 1.086696, "ttft_error_pct": 21.57, "ttft_ms": 98.123, "ttft_relative": 0.784268, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6342.1646009000015, "interactivity_tok_s": 146.36213074128372, "output_per_gpu": 24.213604712320834, "total_per_gpu": 48.25639497854868, "tpot_error_pct": 0.65, "tpot_ms": 6.832368420268798, "tpot_relative": 1.006467, "ttft_error_pct": 57.09, "ttft_ms": 53.68140909999986, "ttft_relative": 0.42906, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 6373.307551792823, "interactivity_tok_s": 147.30871139674315, "output_per_gpu": 36.14103447819887, "total_per_gpu": 48.201120652386926, "tpot_ms": 6.78846478608263, "tpot_relative": 1.0, "ttft_ms": 125.11408070567995, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9017.66, "interactivity_tok_s": 114.69205184080744, "output_per_gpu": 70.342, "total_per_gpu": null, "tpot_error_pct": 12.35, "tpot_ms": 8.719, "tpot_relative": 1.284385, "ttft_error_pct": 40.27, "ttft_ms": 98.123, "ttft_relative": 0.784268, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7450.669716750001, "interactivity_tok_s": 124.05662104619441, "output_per_gpu": 79.60404404817517, "total_per_gpu": 159.98658524455092, "tpot_error_pct": 3.87, "tpot_ms": 8.06083537957748, "tpot_relative": 1.187431, "ttft_error_pct": 62.89, "ttft_ms": 60.96204975000004, "ttft_relative": 0.487252, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 7277.76868136134, "interactivity_tok_s": 128.8554831210933, "output_per_gpu": 122.04334614599735, "total_per_gpu": 164.14074814594287, "tpot_ms": 7.7606321110933205, "tpot_relative": 1.143209, "ttft_ms": 164.26845628302544, "ttft_relative": 1.312949, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9950.636, "interactivity_tok_s": 103.83137784238397, "output_per_gpu": 127.49483333333335, "total_per_gpu": null, "tpot_error_pct": 15.72, "tpot_ms": 9.631, "tpot_relative": 1.41873, "ttft_error_pct": 44.88, "ttft_ms": 98.123, "ttft_relative": 0.784268, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8353.837269437508, "interactivity_tok_s": 112.02201596488212, "output_per_gpu": 145.06639614997695, "total_per_gpu": 289.06127692866613, "tpot_error_pct": 7.26, "tpot_ms": 8.926816674264199, "tpot_relative": 1.314998, "ttft_error_pct": 61.52, "ttft_ms": 68.5087568249991, "ttft_relative": 0.54757, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 7880.6138401065255, "interactivity_tok_s": 120.1493801956883, "output_per_gpu": 226.7831829845981, "total_per_gpu": 302.56730538017183, "tpot_ms": 8.32297260602836, "tpot_relative": 1.226046, "ttft_ms": 178.01980585500132, "ttft_relative": 1.42286, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11031.947, "interactivity_tok_s": 93.562874251497, "output_per_gpu": 229.7105, "total_per_gpu": null, "tpot_error_pct": 15.92, "tpot_ms": 10.688, "tpot_relative": 1.574436, "ttft_error_pct": 59.11, "ttft_ms": 98.123, "ttft_relative": 0.784268, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9149.678222912462, "interactivity_tok_s": 100.90162803572031, "output_per_gpu": 260.5049253832537, "total_per_gpu": 523.7830141629021, "tpot_error_pct": 7.49, "tpot_ms": 9.910642865405391, "tpot_relative": 1.459924, "ttft_error_pct": 65.77, "ttft_ms": 82.13165916250003, "ttft_relative": 0.656454, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 8668.634094059234, "interactivity_tok_s": 108.45544731563078, "output_per_gpu": 411.11444946149635, "total_per_gpu": 552.0105087989301, "tpot_ms": 9.2203759677443, "tpot_relative": 1.358242, "ttft_ms": 239.9598837204394, "ttft_relative": 1.917929, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10309.380137137518, "interactivity_tok_s": 90.36626266097119, "output_per_gpu": 463.9925388600514, "total_per_gpu": 926.5173479816077, "tpot_error_pct": 4.47, "tpot_ms": 11.066076769731186, "tpot_relative": 1.63013, "ttft_error_pct": 64.67, "ttft_ms": 99.24306210312558, "ttft_relative": 0.793221, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 10979.99508463181, "interactivity_tok_s": 86.33068010736831, "output_per_gpu": 660.6021501515218, "total_per_gpu": 880.371246673711, "tpot_ms": 11.583367567084071, "tpot_relative": 1.706331, "ttft_ms": 280.90615201435867, "ttft_relative": 2.2452, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12685.159015010933, "interactivity_tok_s": 73.34592159285454, "output_per_gpu": 750.0294281542664, "total_per_gpu": 1500.3970275230522, "tpot_error_pct": 1.23, "tpot_ms": 13.634023246050825, "tpot_relative": 2.00841, "ttft_error_pct": 60.64, "ttft_ms": 131.43109430000064, "ttft_relative": 1.05049, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 12730.163222424744, "interactivity_tok_s": 74.24892234790327, "output_per_gpu": 1128.824949682813, "total_per_gpu": 1508.8694203505756, "tpot_ms": 13.468208943348243, "tpot_relative": 1.983985, "ttft_ms": 333.8930330475705, "ttft_relative": 2.668709, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 33.33, "points": 2, "status_counts": {"failed": 4, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.12, "tpot_shape_error_pct": 5.9, "ttft_mape_pct": 51.58, "ttft_shape_error_pct": 46.4}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.35, "tpot_shape_error_pct": 5.17, "ttft_mape_pct": 55.93, "ttft_shape_error_pct": 32.86}, "framework": "dynamo-vllm", "id": "720a8caeb227ee75", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7644.794, "interactivity_tok_s": 135.556459265284, "output_per_gpu": 25.0593, "total_per_gpu": null, "tpot_error_pct": 8.9, "tpot_ms": 7.377, "tpot_relative": 1.089034, "ttft_error_pct": 36.95, "ttft_ms": 98.123, "ttft_relative": 0.630531, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6443.487685750003, "interactivity_tok_s": 146.2313111661013, "output_per_gpu": 28.93485780127508, "total_per_gpu": 57.340674096640726, "tpot_error_pct": 0.95, "tpot_ms": 6.838480705846365, "tpot_relative": 1.009534, "ttft_error_pct": 65.14, "ttft_ms": 54.250202599999355, "ttft_relative": 0.348608, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 6485.393137589563, "interactivity_tok_s": 147.62553338984532, "output_per_gpu": 35.90242488910662, "total_per_gpu": 57.13370436883738, "tpot_ms": 6.773895931398456, "tpot_relative": 1.0, "ttft_ms": 155.6195656536147, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11031.947, "interactivity_tok_s": 93.562874251497, "output_per_gpu": 275.6526, "total_per_gpu": null, "tpot_error_pct": 15.33, "tpot_ms": 10.688, "tpot_relative": 1.577822, "ttft_error_pct": 66.21, "ttft_ms": 98.123, "ttft_relative": 0.630531, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9358.579645381262, "interactivity_tok_s": 99.64268910326388, "output_per_gpu": 306.82631395145944, "total_per_gpu": 612.6820560341432, "tpot_error_pct": 8.29, "tpot_ms": 10.035859218569044, "tpot_relative": 1.481549, "ttft_error_pct": 66.14, "ttft_ms": 98.31267534999755, "ttft_relative": 0.63175, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 8850.970128799963, "interactivity_tok_s": 107.90327150181754, "output_per_gpu": 408.18325186991217, "total_per_gpu": 652.7731674581189, "tpot_ms": 9.267559602983454, "tpot_relative": 1.368128, "ttft_ms": 290.3487218245573, "ttft_relative": 1.86576, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10433.35501736718, "interactivity_tok_s": 89.37897107986016, "output_per_gpu": 544.0671989574372, "total_per_gpu": 1088.379705443546, "tpot_error_pct": 3.75, "tpot_ms": 11.188314073413302, "tpot_relative": 1.651681, "ttft_error_pct": 61.87, "ttft_ms": 130.66075202812536, "ttft_relative": 0.839616, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 11041.561082776389, "interactivity_tok_s": 86.02904679047973, "output_per_gpu": 651.541153137394, "total_per_gpu": 1045.0766762501512, "tpot_ms": 11.623980937920418, "tpot_relative": 1.715996, "ttft_ms": 342.6353614406253, "ttft_relative": 2.20175, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12837.851968417173, "interactivity_tok_s": 72.64639543215473, "output_per_gpu": 886.4881864949318, "total_per_gpu": 1774.923844959115, "tpot_error_pct": 1.75, "tpot_ms": 13.76530788693998, "tpot_relative": 2.032111, "ttft_error_pct": 57.27, "ttft_ms": 188.48431433124966, "ttft_relative": 1.211186, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 12872.720667070644, "interactivity_tok_s": 73.9198654231377, "output_per_gpu": 1111.6424877873164, "total_per_gpu": 1784.692797914784, "tpot_ms": 13.528163157166537, "tpot_relative": 1.997102, "ttft_ms": 441.1221465874405, "ttft_relative": 2.834619, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14611.739829705077, "interactivity_tok_s": 64.37600818032759, "output_per_gpu": 1559.3678679425552, "total_per_gpu": 3117.228611898628, "tpot_error_pct": 1.28, "tpot_ms": 15.533737307831183, "tpot_relative": 2.293176, "ttft_error_pct": 44.69, "ttft_ms": 307.00413382031184, "ttft_relative": 1.972786, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 15047.009208596774, "interactivity_tok_s": 63.54883251099734, "output_per_gpu": 1893.8972986227727, "total_per_gpu": 3034.768280199389, "tpot_ms": 15.73593031511549, "tpot_relative": 2.323025, "ttft_ms": 555.032847754137, "ttft_relative": 3.566601, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 18414.086510509842, "interactivity_tok_s": 51.43501083692616, "output_per_gpu": 2476.9814903139622, "total_per_gpu": 4951.6812371720835, "tpot_error_pct": 10.04, "tpot_ms": 19.442010096400743, "tpot_relative": 2.870137, "ttft_error_pct": 40.46, "ttft_ms": 504.07723158085844, "ttft_relative": 3.239164, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 20751.097392618296, "interactivity_tok_s": 46.26842714010061, "output_per_gpu": 2757.5864447713784, "total_per_gpu": 4420.082239035733, "tpot_ms": 21.613010465473657, "tpot_relative": 3.190632, "ttft_ms": 846.5578300198104, "ttft_relative": 5.439919, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 34.86, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5f348f11ba6dc49c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19676.774205014917, "interactivity_tok_s": 49.11029950017996, "output_per_gpu": 3296.4874499941648, "total_per_gpu": 6592.780755298084, "tpot_error_pct": 9.59, "tpot_ms": 20.362327458343753, "tpot_relative": 0.904069, "ttft_error_pct": 34.86, "ttft_ms": 925.0705362125085, "ttft_relative": 0.651442, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 22157.706069357915, "interactivity_tok_s": 44.399119574789935, "output_per_gpu": 3424.356997118588, "total_per_gpu": 5881.838869577155, "tpot_ms": 22.522969139410716, "tpot_relative": 1.0, "ttft_ms": 1420.0345329215224, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.31, "tpot_shape_error_pct": 3.07, "ttft_mape_pct": 52.13, "ttft_shape_error_pct": 26.67}, "framework": "dynamo-vllm", "id": "8fc7c595a7f71da8", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17426.121947231226, "interactivity_tok_s": 53.75922337522479, "output_per_gpu": 2171.8559406499603, "total_per_gpu": 4341.711213108017, "tpot_error_pct": 8.71, "tpot_ms": 18.60145919557415, "tpot_relative": 0.912867, "ttft_error_pct": 57.76, "ttft_ms": 288.9144281892573, "ttft_relative": 0.422418, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 19450.936660070874, "interactivity_tok_s": 49.07501286840144, "output_per_gpu": 2928.1463980722965, "total_per_gpu": 3911.224587947245, "tpot_ms": 20.37696867612811, "tpot_relative": 1.0, "ttft_ms": 683.9539674292155, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22759.563643199715, "interactivity_tok_s": 41.3418644427883, "output_per_gpu": 3338.5346559221066, "total_per_gpu": 6676.872690808496, "tpot_error_pct": 5.91, "tpot_ms": 24.18855592214203, "tpot_relative": 1.187054, "ttft_error_pct": 46.49, "ttft_ms": 484.2208094278275, "ttft_relative": 0.707973, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 24573.701128128912, "interactivity_tok_s": 38.8994075501254, "output_per_gpu": 4635.993610093257, "total_per_gpu": 6193.447316948609, "tpot_ms": 25.707332398608116, "tpot_relative": 1.261588, "ttft_ms": 904.9529169195011, "ttft_relative": 1.32312, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.59, "tpot_shape_error_pct": 56.54, "ttft_mape_pct": 55.76, "ttft_shape_error_pct": 62.81}, "framework": "dynamo-vllm", "id": "8f7346d39f9c143e", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 37266.90840477693, "interactivity_tok_s": 25.95901008576221, "output_per_gpu": 6128.362318068252, "total_per_gpu": 12261.178070743223, "tpot_error_pct": 4.63, "tpot_ms": 38.52227017502767, "tpot_relative": 0.953701, "ttft_error_pct": 35.5, "ttft_ms": 1811.837181862252, "ttft_relative": 0.644977, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 39984.69324071816, "interactivity_tok_s": 24.757141952713777, "output_per_gpu": 7612.148763945709, "total_per_gpu": 11443.64440657077, "tpot_ms": 40.39238462622234, "tpot_relative": 1.0, "ttft_ms": 2809.1494771378107, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 59601.73412274546, "interactivity_tok_s": 18.746775213357637, "output_per_gpu": 7690.688810346634, "total_per_gpu": 15382.989489360125, "tpot_error_pct": 58.55, "tpot_ms": 53.34250763765867, "tpot_relative": 1.320608, "ttft_error_pct": 76.01, "ttft_ms": 10496.679256454094, "ttft_relative": 3.736604, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 162239.50889395268, "interactivity_tok_s": 7.770700834823432, "output_per_gpu": 3636.025193609168, "total_per_gpu": 5465.065548294246, "tpot_ms": 128.68852131311297, "tpot_relative": 3.18596, "ttft_ms": 43761.177183041465, "ttft_relative": 15.578088, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}]}, {"aic": {"coverage_pct": 22.73, "points": 5, "status_counts": {"failed": 17, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.55, "tpot_shape_error_pct": 8.38, "ttft_mape_pct": 41.83, "ttft_shape_error_pct": 27.3}, "aisimulate": {"coverage_pct": 100.0, "points": 22, "status_counts": {"failed": 0, "success": 22, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.04, "tpot_shape_error_pct": 9.05, "ttft_mape_pct": 42.54, "ttft_shape_error_pct": 15.17}, "gpu": "gb300", "precisions": ["fp8"], "rows": 22, "topologies": [{"aic": {"coverage_pct": 40.0, "points": 2, "status_counts": {"failed": 3, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.34, "tpot_shape_error_pct": 10.7, "ttft_mape_pct": 54.6, "ttft_shape_error_pct": 25.32}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.65, "tpot_shape_error_pct": 7.84, "ttft_mape_pct": 42.21, "ttft_shape_error_pct": 20.59}, "framework": "dynamo-vllm", "id": "4bde5687011f0756", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11216.574, "interactivity_tok_s": 92.36168837166343, "output_per_gpu": 302.51233333333334, "total_per_gpu": null, "tpot_error_pct": 18.69, "tpot_ms": 10.827, "tpot_relative": 1.186892, "ttft_error_pct": 48.02, "ttft_ms": 140.553, "ttft_relative": 0.519846, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9517.268909631257, "interactivity_tok_s": 98.33886194980701, "output_per_gpu": 334.8482581649006, "total_per_gpu": 668.6373037234964, "tpot_error_pct": 11.48, "tpot_ms": 10.168919796025385, "tpot_relative": 1.114751, "ttft_error_pct": 50.38, "ttft_ms": 134.1556900937499, "ttft_relative": 0.496185, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 8696.064648458014, "interactivity_tok_s": 109.62339297927709, "output_per_gpu": 415.5685147089809, "total_per_gpu": 738.4264417723984, "tpot_ms": 9.122140565280965, "tpot_relative": 1.0, "ttft_ms": 270.3743635358478, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12636.498, "interactivity_tok_s": 81.86655751125666, "output_per_gpu": 536.1482222222222, "total_per_gpu": null, "tpot_error_pct": 6.0, "tpot_ms": 12.215, "tpot_relative": 1.33905, "ttft_error_pct": 61.18, "ttft_ms": 140.553, "ttft_relative": 0.519846, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10631.457638126547, "interactivity_tok_s": 88.19764276288338, "output_per_gpu": 593.5445971281873, "total_per_gpu": 1187.3568100188338, "tpot_error_pct": 1.61, "tpot_ms": 11.338171505200757, "tpot_relative": 1.242929, "ttft_error_pct": 47.17, "ttft_ms": 191.25906069843705, "ttft_relative": 0.707386, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 10968.8106187421, "interactivity_tok_s": 86.77447915357814, "output_per_gpu": 654.6734625122098, "total_per_gpu": 1166.7788032470019, "tpot_ms": 11.524125638716265, "tpot_relative": 1.263314, "ttft_ms": 362.02225522829394, "ttft_relative": 1.338967, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13341.43207702814, "interactivity_tok_s": 70.32815936031251, "output_per_gpu": 947.3397134485533, "total_per_gpu": 1896.7605798841469, "tpot_error_pct": 6.51, "tpot_ms": 14.21905548354673, "tpot_relative": 1.558741, "ttft_error_pct": 44.26, "ttft_ms": 275.4107877851559, "ttft_relative": 1.018628, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 12761.839314363306, "interactivity_tok_s": 74.9063980958453, "output_per_gpu": 1122.4534016559705, "total_per_gpu": 2002.2769349940572, "tpot_ms": 13.349994465365505, "tpot_relative": 1.463472, "ttft_ms": 494.141632621222, "ttft_relative": 1.82762, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15808.802208084371, "interactivity_tok_s": 59.95503389484963, "output_per_gpu": 1601.2541555406085, "total_per_gpu": 3200.9607041336317, "tpot_error_pct": 7.35, "tpot_ms": 16.67916661933376, "tpot_relative": 1.828427, "ttft_error_pct": 36.69, "ttft_ms": 449.54924729257556, "ttft_relative": 1.662692, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 15019.383274471647, "interactivity_tok_s": 64.36035861882493, "output_per_gpu": 1894.711555005614, "total_per_gpu": 3373.414487938977, "tpot_ms": 15.537514418191996, "tpot_relative": 1.703275, "ttft_ms": 710.0228558762183, "ttft_relative": 2.626073, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19855.094219711747, "interactivity_tok_s": 48.20685594587012, "output_per_gpu": 2550.0274433662007, "total_per_gpu": 5097.705854874929, "tpot_error_pct": 1.28, "tpot_ms": 20.743937358679165, "tpot_relative": 2.274021, "ttft_error_pct": 32.55, "ttft_ms": 746.292503458794, "ttft_relative": 2.760219, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 20458.958852124102, "interactivity_tok_s": 47.589276298701236, "output_per_gpu": 2796.720811972658, "total_per_gpu": 4980.899958923965, "tpot_ms": 21.01313736572395, "tpot_relative": 2.303531, "ttft_ms": 1106.3667460957618, "ttft_relative": 4.091981, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.9, "tpot_shape_error_pct": 9.37, "ttft_mape_pct": 40.73, "ttft_shape_error_pct": 12.35}, "framework": "dynamo-vllm", "id": "dd0179a509daa065", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14087.114815109826, "interactivity_tok_s": 66.65788550654105, "output_per_gpu": 1608.4219250598937, "total_per_gpu": 3215.2893155462107, "tpot_error_pct": 13.31, "tpot_ms": 15.001976021304657, "tpot_relative": 0.866881, "ttft_error_pct": 45.24, "ttft_ms": 270.709786867578, "ttft_relative": 0.54757, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 16432.74581080245, "interactivity_tok_s": 57.78445749104458, "output_per_gpu": 1719.0972838548041, "total_per_gpu": 2754.6699134178457, "tpot_ms": 17.305691589385948, "tpot_relative": 1.0, "ttft_ms": 494.3834671937111, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17271.917952613112, "interactivity_tok_s": 54.69689362729691, "output_per_gpu": 2633.08123214259, "total_per_gpu": 5263.7369250174725, "tpot_error_pct": 8.3, "tpot_ms": 18.282573902897884, "tpot_relative": 1.056449, "ttft_error_pct": 40.69, "ttft_ms": 428.82631298281166, "ttft_relative": 0.867396, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 19085.913201553944, "interactivity_tok_s": 50.15508359977099, "output_per_gpu": 2984.4995612836856, "total_per_gpu": 4783.796906259319, "tpot_ms": 19.93815837253566, "tpot_relative": 1.152116, "ttft_ms": 723.0365417388612, "ttft_relative": 1.462501, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22964.801147815997, "interactivity_tok_s": 41.41091755655404, "output_per_gpu": 3970.3041606350916, "total_per_gpu": 7940.374492540926, "tpot_error_pct": 2.08, "tpot_ms": 24.148221266392383, "tpot_relative": 1.395392, "ttft_error_pct": 36.27, "ttft_ms": 726.8671369825157, "ttft_relative": 1.47025, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 23845.81473053608, "interactivity_tok_s": 40.5508637652773, "output_per_gpu": 4768.382240479061, "total_per_gpu": 7644.374003238462, "tpot_ms": 24.660387156938324, "tpot_relative": 1.424987, "ttft_ms": 1140.517220391331, "ttft_relative": 2.306949, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.1, "tpot_shape_error_pct": 12.16, "ttft_mape_pct": 41.68, "ttft_shape_error_pct": 19.75}, "framework": "dynamo-vllm", "id": "e03530c1a3ad1036", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 18182.33027031755, "interactivity_tok_s": 51.89472593504539, "output_per_gpu": 2500.503779992864, "total_per_gpu": 4998.704148289429, "tpot_error_pct": 6.52, "tpot_ms": 19.26978092632498, "tpot_relative": 0.934754, "ttft_error_pct": 38.83, "ttft_ms": 429.1092439734366, "ttft_relative": 0.611744, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 19687.393275234397, "interactivity_tok_s": 48.50881036938155, "output_per_gpu": 2899.4714104412596, "total_per_gpu": 4647.506919749824, "tpot_ms": 20.61481187407543, "tpot_relative": 1.0, "ttft_ms": 701.4525075049278, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 21645.52667513333, "interactivity_tok_s": 44.04103029387538, "output_per_gpu": 4195.44708536425, "total_per_gpu": 8390.64708233906, "tpot_error_pct": 6.21, "tpot_ms": 22.706099138172664, "tpot_relative": 1.101446, "ttft_error_pct": 35.05, "ttft_ms": 735.4754038659205, "ttft_relative": 1.048503, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 23421.654324457242, "interactivity_tok_s": 41.308110046040184, "output_per_gpu": 4854.309349669854, "total_per_gpu": 7782.127003426959, "tpot_ms": 24.208321292972357, "tpot_relative": 1.174317, "ttft_ms": 1132.3703049833682, "ttft_relative": 1.614322, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 30869.456276104822, "interactivity_tok_s": 31.235208527207156, "output_per_gpu": 5911.21533740046, "total_per_gpu": 11819.838861956858, "tpot_error_pct": 3.6, "tpot_ms": 32.01515364076916, "tpot_relative": 1.553017, "ttft_error_pct": 32.95, "ttft_ms": 1372.4388212770514, "ttft_relative": 1.956567, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 32646.81286020803, "interactivity_tok_s": 30.11099910366714, "output_per_gpu": 6974.639801167078, "total_per_gpu": 11176.511633114247, "tpot_ms": 33.21045563972046, "tpot_relative": 1.611, "ttft_ms": 2046.8520973014593, "ttft_relative": 2.91802, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 47331.999415601575, "interactivity_tok_s": 21.612569082997265, "output_per_gpu": 7736.885808829861, "total_per_gpu": 15479.393954137395, "tpot_error_pct": 11.86, "tpot_ms": 46.269372056592104, "tpot_relative": 2.244472, "ttft_error_pct": 70.13, "ttft_ms": 4746.693956760431, "ttft_relative": 6.76695, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 53961.65187876678, "interactivity_tok_s": 24.17580479115385, "output_per_gpu": 8445.374081706881, "total_per_gpu": 13542.682658966067, "tpot_ms": 41.36366952987266, "tpot_relative": 2.006502, "ttft_ms": 15891.352429637567, "ttft_relative": 22.654923, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 90533.97743675005, "interactivity_tok_s": 20.235211367245647, "output_per_gpu": 8044.76143454415, "total_per_gpu": 16091.208946786728, "tpot_error_pct": 17.3, "tpot_ms": 49.41880674489425, "tpot_relative": 2.397248, "ttft_error_pct": 31.46, "ttft_ms": 45040.152788572115, "ttft_relative": 64.209839, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 104493.8351453994, "interactivity_tok_s": 23.735071271613258, "output_per_gpu": 8665.422567923364, "total_per_gpu": 13892.709704536708, "tpot_ms": 42.13174624826103, "tpot_relative": 2.043761, "ttft_ms": 65711.1379181547, "ttft_relative": 93.67867, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 60.0, "points": 3, "status_counts": {"failed": 2, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.02, "tpot_shape_error_pct": 7.23, "ttft_mape_pct": 33.33, "ttft_shape_error_pct": 28.3}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.91, "tpot_shape_error_pct": 6.45, "ttft_mape_pct": 47.6, "ttft_shape_error_pct": 5.29}, "framework": "dynamo-vllm", "id": "a744e3e02e28f486", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10292.805, "interactivity_tok_s": 100.76582023377671, "output_per_gpu": 148.4718, "total_per_gpu": null, "tpot_error_pct": 21.89, "tpot_ms": 9.924, "tpot_relative": 1.218908, "ttft_error_pct": 17.82, "ttft_ms": 140.553, "ttft_relative": 0.821778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8623.995723999993, "interactivity_tok_s": 108.80312997301752, "output_per_gpu": 168.5325741242363, "total_per_gpu": 335.82030279473895, "tpot_error_pct": 12.89, "tpot_ms": 9.190912065195125, "tpot_relative": 1.128867, "ttft_error_pct": 45.29, "ttft_ms": 93.57969505000032, "ttft_relative": 0.547137, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 7704.726784552622, "interactivity_tok_s": 122.82428022282467, "output_per_gpu": 231.95064049771162, "total_per_gpu": 371.3538862254906, "tpot_ms": 8.141712682425865, "tpot_relative": 1.0, "ttft_ms": 171.03519500160473, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11216.574, "interactivity_tok_s": 92.36168837166343, "output_per_gpu": 272.26120000000003, "total_per_gpu": null, "tpot_error_pct": 19.99, "tpot_ms": 10.827, "tpot_relative": 1.329818, "ttft_error_pct": 34.51, "ttft_ms": 140.553, "ttft_relative": 0.821778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9319.645212518779, "interactivity_tok_s": 99.32633060522588, "output_per_gpu": 306.6447608493627, "total_per_gpu": 616.5538593124293, "tpot_error_pct": 11.58, "tpot_ms": 10.067823847983636, "tpot_relative": 1.236573, "ttft_error_pct": 49.52, "ttft_ms": 108.34420014375094, "ttft_relative": 0.633461, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 8462.930674257223, "interactivity_tok_s": 110.82785732733639, "output_per_gpu": 423.10698391474, "total_per_gpu": 681.7357116192456, "tpot_ms": 9.023002195616243, "tpot_relative": 1.108244, "ttft_ms": 214.60996227651776, "ttft_relative": 1.254771, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12636.498, "interactivity_tok_s": 81.86655751125666, "output_per_gpu": 482.5334, "total_per_gpu": null, "tpot_error_pct": 6.16, "tpot_ms": 12.215, "tpot_relative": 1.500299, "ttft_error_pct": 47.65, "ttft_ms": 140.553, "ttft_relative": 0.821778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10525.865904218783, "interactivity_tok_s": 88.79537993810564, "output_per_gpu": 545.1930375781192, "total_per_gpu": 1088.661487005662, "tpot_error_pct": 2.12, "tpot_ms": 11.261847189538969, "tpot_relative": 1.383228, "ttft_error_pct": 49.7, "ttft_ms": 135.03094746250122, "ttft_relative": 0.789492, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 10895.558569153218, "interactivity_tok_s": 86.91324591684877, "output_per_gpu": 664.3953640791499, "total_per_gpu": 1062.511664228342, "tpot_ms": 11.505726077204795, "tpot_relative": 1.413183, "ttft_ms": 268.4641320642186, "ttft_relative": 1.569643, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13181.67939173902, "interactivity_tok_s": 70.85748997164136, "output_per_gpu": 866.5319428089181, "total_per_gpu": 1733.4545851671064, "tpot_error_pct": 6.42, "tpot_ms": 14.112834089949008, "tpot_relative": 1.733399, "ttft_error_pct": 47.88, "ttft_ms": 187.16602044843756, "ttft_relative": 1.094313, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 12565.0314012536, "interactivity_tok_s": 75.40657292717682, "output_per_gpu": 1142.4628380192712, "total_per_gpu": 1832.5185749927873, "tpot_ms": 13.261443415095131, "tpot_relative": 1.628827, "ttft_ms": 359.11660954261606, "ttft_relative": 2.099665, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15389.71863325775, "interactivity_tok_s": 60.78755864757463, "output_per_gpu": 1479.2325926334013, "total_per_gpu": 2961.7148213635305, "tpot_error_pct": 6.55, "tpot_ms": 16.450734693881298, "tpot_relative": 2.02055, "ttft_error_pct": 45.62, "ttft_ms": 273.06083879687395, "ttft_relative": 1.596518, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 14689.897599476899, "interactivity_tok_s": 64.76976638855402, "output_per_gpu": 1951.2393859243548, "total_per_gpu": 3132.6284460378783, "tpot_ms": 15.439302251007005, "tpot_relative": 1.896321, "ttft_ms": 502.1621536863222, "ttft_relative": 2.936016, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.64, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.29, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "491bfef6d95e4754", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 29235.92992910662, "interactivity_tok_s": 32.31187414075773, "output_per_gpu": 3901.4940890112357, "total_per_gpu": 7802.758401821927, "tpot_error_pct": 2.64, "tpot_ms": 30.948375066199407, "tpot_relative": 0.973561, "ttft_error_pct": 36.29, "ttft_ms": 735.7860596384778, "ttft_relative": 0.637081, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 30423.248484085558, "interactivity_tok_s": 31.45756902424695, "output_per_gpu": 5002.3408608290665, "total_per_gpu": 7518.226800388876, "tpot_ms": 31.788851809534847, "tpot_relative": 1.0, "ttft_ms": 1154.9333016733897, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.11, "tpot_shape_error_pct": 10.13, "ttft_mape_pct": 40.0, "ttft_shape_error_pct": 17.79}, "framework": "dynamo-vllm", "id": "1ea623bdabf54d6d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16104.0422154445, "interactivity_tok_s": 58.81971195627964, "output_per_gpu": 1572.0749180295156, "total_per_gpu": 3142.630430749856, "tpot_error_pct": 6.19, "tpot_ms": 17.001103316236815, "tpot_relative": 1.061882, "ttft_error_pct": 34.6, "ttft_ms": 448.3533142039052, "ttft_relative": 0.654024, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 15430.401840205923, "interactivity_tok_s": 62.45959361227212, "output_per_gpu": 1844.8392107422653, "total_per_gpu": 3284.6199227498905, "tpot_ms": 16.010350727026168, "tpot_relative": 1.0, "ttft_ms": 685.5303047435882, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19010.533987104536, "interactivity_tok_s": 50.45708802428694, "output_per_gpu": 2659.693671811824, "total_per_gpu": 5316.937289534051, "tpot_error_pct": 3.3, "tpot_ms": 19.818821084535468, "tpot_relative": 1.237876, "ttft_error_pct": 31.07, "ttft_ms": 753.5296325107438, "ttft_relative": 1.099192, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 19968.472898284017, "interactivity_tok_s": 48.79299525911234, "output_per_gpu": 2860.752483160919, "total_per_gpu": 5094.938996008429, "tpot_ms": 20.494745089731808, "tpot_relative": 1.280093, "ttft_ms": 1093.1526177934984, "ttft_relative": 1.594609, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 23796.190813374888, "interactivity_tok_s": 41.85438586680328, "output_per_gpu": 4244.419698601316, "total_per_gpu": 8488.589424599928, "tpot_error_pct": 5.83, "tpot_ms": 23.89235869288308, "tpot_relative": 1.492307, "ttft_error_pct": 54.34, "ttft_ms": 1793.5619365233524, "ttft_relative": 2.616313, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 27288.486402453895, "interactivity_tok_s": 39.414028011590744, "output_per_gpu": 4176.733195404309, "total_per_gpu": 7439.865590371725, "tpot_ms": 25.371677304992104, "tpot_relative": 1.584705, "ttft_ms": 3927.7778420856976, "ttft_relative": 5.729547, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}]}, {"aic": {"coverage_pct": 100.0, "points": 11, "status_counts": {"failed": 0, "success": 11, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.55, "tpot_shape_error_pct": 6.93, "ttft_mape_pct": 15.96, "ttft_shape_error_pct": 19.25}, "aisimulate": {"coverage_pct": 100.0, "points": 11, "status_counts": {"failed": 0, "success": 11, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.81, "tpot_shape_error_pct": 5.86, "ttft_mape_pct": 46.09, "ttft_shape_error_pct": 24.29}, "gpu": "h100", "precisions": ["fp8"], "rows": 11, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.16, "tpot_shape_error_pct": 4.82, "ttft_mape_pct": 16.9, "ttft_shape_error_pct": 19.22}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.34, "tpot_shape_error_pct": 4.78, "ttft_mape_pct": 38.77, "ttft_shape_error_pct": 26.09}, "framework": "vllm", "id": "46af50cb4408a7ed", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11043.737819380558, "interactivity_tok_s": 93.64412453374041, "output_per_gpu": 92.63168111477134, "total_per_gpu": null, "tpot_error_pct": 7.4, "tpot_ms": 10.678726561640238, "tpot_relative": 1.073988, "ttft_error_pct": 8.95, "ttft_ms": 119.4005468225945, "ttft_relative": 1.089546, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9768.238797724967, "interactivity_tok_s": 94.38923544797903, "output_per_gpu": 91.35284170431477, "total_per_gpu": 183.59907931077478, "tpot_error_pct": 6.55, "tpot_ms": 10.594428435127355, "tpot_relative": 1.065509, "ttft_error_pct": 49.34, "ttft_ms": 55.51614032499954, "ttft_relative": 0.506592, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 9226.175266579958, "interactivity_tok_s": 100.57262437473588, "output_per_gpu": 96.45168016738208, "total_per_gpu": 193.84662092971607, "tpot_ms": 9.943063594263757, "tpot_relative": 1.0, "ttft_ms": 109.58740237983875, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13078.898096306686, "interactivity_tok_s": 79.01223923126796, "output_per_gpu": 156.43519698175217, "total_per_gpu": null, "tpot_error_pct": 6.77, "tpot_ms": 12.656267050893355, "tpot_relative": 1.272874, "ttft_error_pct": 47.92, "ttft_ms": 131.53690324278463, "ttft_relative": 1.200292, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11775.83981829993, "interactivity_tok_s": 79.26860526795689, "output_per_gpu": 154.76877143672533, "total_per_gpu": 308.3943620818197, "tpot_error_pct": 6.42, "tpot_ms": 12.61533486832062, "tpot_relative": 1.268757, "ttft_error_pct": 27.7, "ttft_ms": 64.29484613749966, "ttft_relative": 0.586699, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 11091.368178173434, "interactivity_tok_s": 84.36102259799479, "output_per_gpu": 163.9685656096444, "total_per_gpu": 326.7259972618621, "tpot_ms": 11.853815532385088, "tpot_relative": 1.192169, "ttft_ms": 88.9265488483943, "ttft_relative": 0.811467, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15603.711743614187, "interactivity_tok_s": 66.17145156858031, "output_per_gpu": 262.2452956857941, "total_per_gpu": null, "tpot_error_pct": 1.78, "tpot_ms": 15.112257269490252, "tpot_relative": 1.519879, "ttft_error_pct": 4.84, "ttft_ms": 143.87255692565773, "ttft_relative": 1.312857, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13938.67316852505, "interactivity_tok_s": 66.01474251919666, "output_per_gpu": 257.384808100614, "total_per_gpu": 517.5095648895785, "tpot_error_pct": 1.55, "tpot_ms": 15.148131490616757, "tpot_relative": 1.523487, "ttft_error_pct": 44.42, "ttft_ms": 76.27712892499997, "ttft_relative": 0.696039, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 14217.225072825386, "interactivity_tok_s": 64.9922889461818, "output_per_gpu": 252.28658663397016, "total_per_gpu": 507.25884965745735, "tpot_ms": 15.38644070265121, "tpot_relative": 1.547455, "ttft_ms": 137.22834836080438, "ttft_relative": 1.252227, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20137.39332012694, "interactivity_tok_s": 51.200937531006076, "output_per_gpu": 406.4081120082334, "total_per_gpu": null, "tpot_error_pct": 2.99, "tpot_ms": 19.530892366852143, "tpot_relative": 1.964273, "ttft_error_pct": 6.62, "ttft_ms": 157.2904288371961, "ttft_relative": 1.435297, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18199.478724371827, "interactivity_tok_s": 50.98393467355536, "output_per_gpu": 397.0616772218016, "total_per_gpu": 792.8673445234716, "tpot_error_pct": 2.58, "tpot_ms": 19.61402167963089, "tpot_relative": 1.972634, "ttft_error_pct": 41.54, "ttft_ms": 98.47345202187446, "ttft_relative": 0.898584, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 18748.329141509566, "interactivity_tok_s": 49.66941617654455, "output_per_gpu": 384.553240511943, "total_per_gpu": 767.8900385601327, "tpot_ms": 20.133113633661576, "tpot_relative": 2.02484, "ttft_ms": 168.43901795655256, "ttft_relative": 1.537029, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27926.93807325342, "interactivity_tok_s": 36.85861733035345, "output_per_gpu": 586.1007732772606, "total_per_gpu": null, "tpot_error_pct": 6.88, "tpot_ms": 27.130697579816427, "tpot_relative": 2.728605, "ttft_error_pct": 16.15, "ttft_ms": 172.23444910121412, "ttft_relative": 1.571663, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25765.281514279755, "interactivity_tok_s": 35.951135943067186, "output_per_gpu": 559.7216625245765, "total_per_gpu": 1119.6956908194861, "tpot_error_pct": 9.58, "tpot_ms": 27.815532771582422, "tpot_relative": 2.797481, "ttft_error_pct": 30.85, "ttft_ms": 142.0415989593753, "ttft_relative": 1.296149, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 23586.049180201735, "interactivity_tok_s": 39.39439405323124, "output_per_gpu": 609.5588862707056, "total_per_gpu": 1219.3926087827017, "tpot_ms": 25.384322415234035, "tpot_relative": 2.552968, "ttft_ms": 205.41815173710347, "ttft_relative": 1.874469, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.88, "tpot_shape_error_pct": 8.62, "ttft_mape_pct": 15.19, "ttft_shape_error_pct": 19.28}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.37, "tpot_shape_error_pct": 6.72, "ttft_mape_pct": 52.19, "ttft_shape_error_pct": 22.85}, "framework": "vllm", "id": "22eb0cadcc7e4e98", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9240.759626740593, "interactivity_tok_s": 111.94509956675245, "output_per_gpu": 55.35259228254773, "total_per_gpu": null, "tpot_error_pct": 5.0, "tpot_ms": 8.932950203896185, "tpot_relative": 1.049966, "ttft_error_pct": 5.12, "ttft_ms": 102.35156815479579, "ttft_relative": 1.051183, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8184.717420949964, "interactivity_tok_s": 112.54700159599523, "output_per_gpu": 54.45441389182616, "total_per_gpu": 109.44137115413818, "tpot_error_pct": 4.44, "tpot_ms": 8.885176733447363, "tpot_relative": 1.044351, "ttft_error_pct": 59.84, "ttft_ms": 39.10595514999939, "ttft_relative": 0.40163, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 7897.2643950372, "interactivity_tok_s": 117.53858863509144, "output_per_gpu": 56.44086351330988, "total_per_gpu": 113.43369711573355, "tpot_ms": 8.507844203443561, "tpot_relative": 1.0, "ttft_ms": 97.36801093677059, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11000.357054020686, "interactivity_tok_s": 93.95968284146166, "output_per_gpu": 92.99698136853551, "total_per_gpu": null, "tpot_error_pct": 6.84, "tpot_ms": 10.642862659373828, "tpot_relative": 1.250947, "ttft_error_pct": 6.71, "ttft_ms": 112.70855348125889, "ttft_relative": 1.157552, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9890.4254357375, "interactivity_tok_s": 94.30643788248801, "output_per_gpu": 92.13917831966715, "total_per_gpu": 183.59778175440175, "tpot_error_pct": 6.45, "tpot_ms": 10.603729951565612, "tpot_relative": 1.246347, "ttft_error_pct": 55.92, "ttft_ms": 46.560507225000265, "ttft_relative": 0.478191, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 9353.099454988842, "interactivity_tok_s": 100.3878643615508, "output_per_gpu": 97.31192317217695, "total_per_gpu": 193.90506360585692, "tpot_ms": 9.961363421363972, "tpot_relative": 1.170845, "ttft_ms": 105.62592554197181, "ttft_relative": 1.084811, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12390.755149570301, "interactivity_tok_s": 83.39068078166937, "output_per_gpu": 165.12310793833686, "total_per_gpu": null, "tpot_error_pct": 0.85, "tpot_ms": 11.991747646456632, "tpot_relative": 1.409493, "ttft_error_pct": 3.39, "ttft_ms": 123.19730724516629, "ttft_relative": 1.265275, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11070.969787937498, "interactivity_tok_s": 83.06875615656458, "output_per_gpu": 161.84177658554086, "total_per_gpu": 325.4064138431946, "tpot_error_pct": 0.47, "tpot_ms": 12.038220460593404, "tpot_relative": 1.414955, "ttft_error_pct": 56.81, "ttft_ms": 55.075151043750246, "ttft_relative": 0.565639, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 11195.981569765718, "interactivity_tok_s": 82.68171494308739, "output_per_gpu": 159.63976735816019, "total_per_gpu": 320.9789542524206, "tpot_ms": 12.094572550754828, "tpot_relative": 1.421579, "ttft_ms": 127.52081859798636, "ttft_relative": 1.309679, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14856.513406034168, "interactivity_tok_s": 69.4847385785, "output_per_gpu": 275.4347462398533, "total_per_gpu": null, "tpot_error_pct": 4.37, "tpot_ms": 14.391649453645933, "tpot_relative": 1.691574, "ttft_error_pct": 12.22, "ttft_ms": 133.85601495437902, "ttft_relative": 1.374743, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13519.82888876252, "interactivity_tok_s": 68.61837390912791, "output_per_gpu": 266.62040651461854, "total_per_gpu": 532.3974229598782, "tpot_error_pct": 3.16, "tpot_ms": 14.573356129428413, "tpot_relative": 1.712932, "ttft_error_pct": 53.58, "ttft_ms": 70.79608489374992, "ttft_relative": 0.727098, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 14042.242569304653, "interactivity_tok_s": 66.45015316185727, "output_per_gpu": 256.8429057417453, "total_per_gpu": 512.8733502059778, "tpot_ms": 15.048874267666928, "tpot_relative": 1.768823, "ttft_ms": 152.4982667833683, "ttft_relative": 1.566205, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18546.601472288006, "interactivity_tok_s": 55.59545148920227, "output_per_gpu": 441.26682789989223, "total_per_gpu": null, "tpot_error_pct": 6.85, "tpot_ms": 17.987082993547048, "tpot_relative": 2.114176, "ttft_error_pct": 24.83, "ttft_ms": 145.8155698893759, "ttft_relative": 1.497572, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17049.28453820941, "interactivity_tok_s": 54.35513052692259, "output_per_gpu": 420.89354605034106, "total_per_gpu": 841.9768634300572, "tpot_error_pct": 4.72, "tpot_ms": 18.39752734113463, "tpot_relative": 2.162419, "ttft_error_pct": 47.84, "ttft_ms": 101.17708734999995, "ttft_relative": 1.03912, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 17984.873798575427, "interactivity_tok_s": 51.788976220013524, "output_per_gpu": 399.67051337074867, "total_per_gpu": 799.5212290880196, "tpot_ms": 19.309128563417254, "tpot_relative": 2.269568, "ttft_ms": 193.98907024224172, "ttft_relative": 1.992329, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22243.005790437073, "interactivity_tok_s": 46.32409589075442, "output_per_gpu": 735.8717681509164, "total_per_gpu": null, "tpot_error_pct": 11.38, "tpot_ms": 21.587037604755167, "tpot_relative": 2.53731, "ttft_error_pct": 38.84, "ttft_ms": 159.4663207725386, "ttft_relative": 1.637769, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20986.49512727574, "interactivity_tok_s": 44.147046218476646, "output_per_gpu": 682.2003748645451, "total_per_gpu": 1365.8994342323936, "tpot_error_pct": 7.01, "tpot_ms": 22.651572090489598, "tpot_relative": 2.662434, "ttft_error_pct": 39.14, "ttft_ms": 158.67699834843762, "ttft_relative": 1.629663, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 22658.5266545806, "interactivity_tok_s": 41.05258457005528, "output_per_gpu": 632.9436609734709, "total_per_gpu": 1267.2777973719274, "tpot_ms": 24.3590022521852, "tpot_relative": 2.863123, "ttft_ms": 260.74281716155383, "ttft_relative": 2.67791, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.25, "tpot_shape_error_pct": 12.48, "ttft_mape_pct": 18.16, "ttft_shape_error_pct": 24.2}, "aisimulate": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.69, "tpot_shape_error_pct": 11.78, "ttft_mape_pct": 47.17, "ttft_shape_error_pct": 46.95}, "gpu": "h200", "precisions": ["fp8"], "rows": 15, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.63, "tpot_shape_error_pct": 12.31, "ttft_mape_pct": 18.63, "ttft_shape_error_pct": 19.29}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.63, "tpot_shape_error_pct": 12.33, "ttft_mape_pct": 42.97, "ttft_shape_error_pct": 60.66}, "framework": "vllm", "id": "27ed2205abf1f24c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7600.133697201109, "interactivity_tok_s": 136.224095904611, "output_per_gpu": 33.65072381479088, "total_per_gpu": null, "tpot_error_pct": 1.4, "tpot_ms": 7.340845195994076, "tpot_relative": 0.986026, "ttft_error_pct": 13.17, "ttft_ms": 90.44906169916908, "ttft_relative": 0.868284, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6791.578760300004, "interactivity_tok_s": 136.30768533137842, "output_per_gpu": 33.917003414066976, "total_per_gpu": 67.59473992755719, "tpot_error_pct": 1.46, "tpot_ms": 7.336343490602852, "tpot_relative": 0.985421, "ttft_error_pct": 62.42, "ttft_ms": 39.1506633999987, "ttft_relative": 0.375835, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 6956.20541870594, "interactivity_tok_s": 134.32045357103286, "output_per_gpu": 33.11178780344081, "total_per_gpu": 65.98998908558538, "tpot_ms": 7.444882543306547, "tpot_relative": 1.0, "ttft_ms": 104.1699094697833, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8491.639146585458, "interactivity_tok_s": 121.98750413964552, "output_per_gpu": 60.235720238498054, "total_per_gpu": null, "tpot_error_pct": 3.47, "tpot_ms": 8.197560947351192, "tpot_relative": 1.1011, "ttft_error_pct": 24.28, "ttft_ms": 105.53429744518814, "ttft_relative": 1.013098, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7671.08663135001, "interactivity_tok_s": 122.54720559109403, "output_per_gpu": 60.7818244365446, "total_per_gpu": 120.45232120897275, "tpot_error_pct": 3.0, "tpot_ms": 8.160120789181617, "tpot_relative": 1.096071, "ttft_error_pct": 66.09, "ttft_ms": 47.261689999999234, "ttft_relative": 0.453698, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 7507.119986141333, "interactivity_tok_s": 126.22604486323907, "output_per_gpu": 61.50360249388682, "total_per_gpu": 122.25575119790471, "tpot_ms": 7.922295284491093, "tpot_relative": 1.064126, "ttft_ms": 139.3834167101886, "ttft_relative": 1.338039, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9563.0272473396, "interactivity_tok_s": 108.30332907941956, "output_per_gpu": 106.97449390668577, "total_per_gpu": null, "tpot_error_pct": 7.51, "tpot_ms": 9.233326514521943, "tpot_relative": 1.240225, "ttft_error_pct": 8.38, "ttft_ms": 117.33422298365278, "ttft_relative": 1.126373, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8452.869142774995, "interactivity_tok_s": 109.12552209269312, "output_per_gpu": 105.41657742272785, "total_per_gpu": 211.86408871166725, "tpot_error_pct": 6.7, "tpot_ms": 9.163759135563014, "tpot_relative": 1.23088, "ttft_error_pct": 59.61, "ttft_ms": 51.7296762249998, "ttft_relative": 0.496589, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 8003.611694020218, "interactivity_tok_s": 116.44102338644892, "output_per_gpu": 110.86752737459838, "total_per_gpu": 222.81929682409614, "tpot_ms": 8.588038570231058, "tpot_relative": 1.153549, "ttft_ms": 128.06458217091858, "ttft_relative": 1.229382, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11139.177861438618, "interactivity_tok_s": 92.91581664911344, "output_per_gpu": 183.67603295776448, "total_per_gpu": null, "tpot_error_pct": 5.55, "tpot_ms": 10.762430295117484, "tpot_relative": 1.445615, "ttft_error_pct": 4.56, "ttft_ms": 129.21166953343214, "ttft_relative": 1.240393, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10013.271935574952, "interactivity_tok_s": 93.26236422676583, "output_per_gpu": 181.85657880535783, "total_per_gpu": 362.36989600960516, "tpot_error_pct": 5.16, "tpot_ms": 10.722438877578927, "tpot_relative": 1.440243, "ttft_error_pct": 52.15, "ttft_ms": 59.13110974999959, "ttft_relative": 0.567641, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 9589.912289055064, "interactivity_tok_s": 98.0731180444786, "output_per_gpu": 189.83417538490002, "total_per_gpu": 378.26616361744, "tpot_ms": 10.196474017951331, "tpot_relative": 1.369595, "ttft_ms": 123.58140678843483, "ttft_relative": 1.186345, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14037.230175957595, "interactivity_tok_s": 73.6187590969346, "output_per_gpu": 291.51050091125626, "total_per_gpu": null, "tpot_error_pct": 42.71, "tpot_ms": 13.583494373808849, "tpot_relative": 1.824541, "ttft_error_pct": 23.73, "ttft_ms": 141.31543155114113, "ttft_relative": 1.356586, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12491.712056375003, "interactivity_tok_s": 73.69420638415626, "output_per_gpu": 287.40223315219663, "total_per_gpu": 577.86396067613, "tpot_error_pct": 42.77, "tpot_ms": 13.56958774733468, "tpot_relative": 1.822673, "ttft_error_pct": 35.9, "ttft_ms": 73.21288243124968, "ttft_relative": 0.702822, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 21819.52764988091, "interactivity_tok_s": 42.1755060843211, "output_per_gpu": 161.6127763829227, "total_per_gpu": 324.94597565303457, "tpot_ms": 23.710444588399465, "tpot_relative": 3.184798, "ttft_ms": 114.21249396516941, "ttft_relative": 1.096406, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16701.843895491606, "interactivity_tok_s": 61.82031147512084, "output_per_gpu": 490.0057772788272, "total_per_gpu": null, "tpot_error_pct": 4.19, "tpot_ms": 16.175913322637385, "tpot_relative": 2.172756, "ttft_error_pct": 8.13, "ttft_ms": 153.8845664335588, "ttft_relative": 1.477246, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15117.437090009427, "interactivity_tok_s": 61.46352793224732, "output_per_gpu": 477.46439003837764, "total_per_gpu": 953.4184353499769, "tpot_error_pct": 3.63, "tpot_ms": 16.26981127901287, "tpot_relative": 2.185368, "ttft_error_pct": 39.42, "ttft_ms": 101.46839938125052, "ttft_relative": 0.974066, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 15751.592535179225, "interactivity_tok_s": 59.23198816618086, "output_per_gpu": 457.92457204983594, "total_per_gpu": 914.4006089270235, "tpot_ms": 16.882769445361294, "tpot_relative": 2.267701, "ttft_ms": 167.5017594534438, "ttft_relative": 1.607967, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22486.766183502838, "interactivity_tok_s": 45.834279290550285, "output_per_gpu": 727.8947922715628, "total_per_gpu": null, "tpot_error_pct": 0.96, "tpot_ms": 21.81773152056896, "tpot_relative": 2.930568, "ttft_error_pct": 23.46, "ttft_ms": 167.22683796079153, "ttft_relative": 1.605328, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20669.618174576644, "interactivity_tok_s": 44.86153795460975, "output_per_gpu": 696.4438579067009, "total_per_gpu": 1393.2017265127586, "tpot_error_pct": 3.15, "tpot_ms": 22.290809579729196, "tpot_relative": 2.994112, "ttft_error_pct": 38.51, "ttft_ms": 134.34330496562524, "ttft_relative": 1.289656, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 20131.37295123597, "interactivity_tok_s": 46.274259375510304, "output_per_gpu": 714.9044516988976, "total_per_gpu": 1430.1312375591244, "tpot_ms": 21.610286442082515, "tpot_relative": 2.902703, "ttft_ms": 218.47782725963043, "ttft_relative": 2.097322, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29241.710106839335, "interactivity_tok_s": 35.20502395226033, "output_per_gpu": 1119.4967695252333, "total_per_gpu": null, "tpot_error_pct": 19.23, "tpot_ms": 28.405036774184474, "tpot_relative": 3.815377, "ttft_error_pct": 6.92, "ttft_ms": 183.3574868486188, "ttft_relative": 1.760177, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27579.91125284504, "interactivity_tok_s": 33.57520909403333, "output_per_gpu": 1042.2527864441063, "total_per_gpu": 2086.795234631478, "tpot_error_pct": 25.02, "tpot_ms": 29.783880040756337, "tpot_relative": 4.000584, "ttft_error_pct": 1.12, "ttft_ms": 194.7781684140622, "ttft_relative": 1.869812, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 22079.143716174287, "interactivity_tok_s": 41.975372822144536, "output_per_gpu": 1247.3500795115838, "total_per_gpu": 2499.342866754221, "tpot_ms": 23.823492985688024, "tpot_relative": 3.199982, "ttft_ms": 196.98296637278142, "ttft_relative": 1.890978, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36703.450311466804, "interactivity_tok_s": 28.02768064382684, "output_per_gpu": 1783.8104985880686, "total_per_gpu": null, "tpot_error_pct": 10.69, "tpot_ms": 35.679013640404534, "tpot_relative": 4.792421, "ttft_error_pct": 55.04, "ttft_ms": 203.81935733296103, "ttft_relative": 1.956605, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35367.13684727006, "interactivity_tok_s": 26.30109207506933, "output_per_gpu": 1630.3773139004415, "total_per_gpu": 3259.1788734152055, "tpot_error_pct": 4.82, "tpot_ms": 38.021234903317755, "tpot_relative": 5.10703, "ttft_error_pct": 31.49, "ttft_ms": 310.5701085484363, "ttft_relative": 2.98138, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 37279.939094719884, "interactivity_tok_s": 25.03280786644124, "output_per_gpu": 1547.6895383255346, "total_per_gpu": 3093.8832397322826, "tpot_ms": 39.94757621020178, "tpot_relative": 5.365777, "ttft_ms": 453.3085410897911, "ttft_relative": 4.351627, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.17, "tpot_shape_error_pct": 12.76, "ttft_mape_pct": 17.45, "ttft_shape_error_pct": 32.07}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.77, "tpot_shape_error_pct": 10.9, "ttft_mape_pct": 53.46, "ttft_shape_error_pct": 25.0}, "framework": "vllm", "id": "4177ae801a696086", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 8908.802188122023, "interactivity_tok_s": 116.124290882694, "output_per_gpu": 57.41512598427379, "total_per_gpu": null, "tpot_error_pct": 8.32, "tpot_ms": 8.611462704303412, "tpot_relative": 1.083241, "ttft_error_pct": 23.22, "ttft_ms": 99.27584161963212, "ttft_relative": 1.232248, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7885.651875500026, "interactivity_tok_s": 116.79314129302409, "output_per_gpu": 56.519289826459776, "total_per_gpu": 113.59131672142317, "tpot_error_pct": 7.7, "tpot_ms": 8.562146620331795, "tpot_relative": 1.077038, "ttft_error_pct": 55.07, "ttft_ms": 36.19396914999976, "ttft_relative": 0.449253, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 7369.173028937075, "interactivity_tok_s": 125.79061662399138, "output_per_gpu": 60.23392791454628, "total_per_gpu": 121.05692063939308, "tpot_ms": 7.94971856278567, "tpot_relative": 1.0, "ttft_ms": 80.56479261722416, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9953.48921259161, "interactivity_tok_s": 103.91931594879328, "output_per_gpu": 102.77802870433206, "total_per_gpu": null, "tpot_error_pct": 3.6, "tpot_ms": 9.622850101253116, "tpot_relative": 1.210464, "ttft_error_pct": 7.02, "ttft_ms": 109.31355900967438, "ttft_relative": 1.35684, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8954.58074991252, "interactivity_tok_s": 104.15947101513729, "output_per_gpu": 101.65919665227867, "total_per_gpu": 202.56749995684453, "tpot_error_pct": 3.36, "tpot_ms": 9.600663197057443, "tpot_relative": 1.207673, "ttft_error_pct": 64.05, "ttft_ms": 42.26605008749933, "ttft_relative": 0.524622, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 8740.697607956827, "interactivity_tok_s": 107.66184581472928, "output_per_gpu": 103.84591633533809, "total_per_gpu": 206.92478738278095, "tpot_ms": 9.288341588726407, "tpot_relative": 1.168386, "ttft_ms": 117.56385122425854, "ttft_relative": 1.459246, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11215.415412395247, "interactivity_tok_s": 92.1957235562704, "output_per_gpu": 182.42748260031155, "total_per_gpu": null, "tpot_error_pct": 2.8, "tpot_ms": 10.846490069462536, "tpot_relative": 1.364387, "ttft_error_pct": 13.15, "ttft_ms": 119.45607133507087, "ttft_relative": 1.482733, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10003.92844142501, "interactivity_tok_s": 91.93150343009951, "output_per_gpu": 179.04191896704359, "total_per_gpu": 359.9897999629018, "tpot_error_pct": 2.52, "tpot_ms": 10.877663942049573, "tpot_relative": 1.368308, "ttft_error_pct": 63.62, "ttft_ms": 50.032635493750675, "ttft_relative": 0.621024, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 10351.392166677397, "interactivity_tok_s": 89.61225519756394, "output_per_gpu": 172.83540663251134, "total_per_gpu": 347.5107048623471, "tpot_ms": 11.159187968156218, "tpot_relative": 1.403721, "ttft_ms": 137.5436806876678, "ttft_relative": 1.707243, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12980.525061859844, "interactivity_tok_s": 79.60623118171173, "output_per_gpu": 315.2414852634397, "total_per_gpu": null, "tpot_error_pct": 9.86, "tpot_ms": 12.561830715454523, "tpot_relative": 1.58016, "ttft_error_pct": 20.78, "ttft_ms": 129.7722399498665, "ttft_relative": 1.610781, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11792.835106206247, "interactivity_tok_s": 78.68524997172398, "output_per_gpu": 305.2597563990652, "total_per_gpu": 609.5538963605613, "tpot_error_pct": 8.81, "tpot_ms": 12.708862211905739, "tpot_relative": 1.598656, "ttft_error_pct": 60.8, "ttft_ms": 64.22236531562524, "ttft_relative": 0.797152, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 13029.945701590623, "interactivity_tok_s": 71.75323539998078, "output_per_gpu": 276.2880167729023, "total_per_gpu": 551.7020622970347, "tpot_ms": 13.936653788858527, "tpot_relative": 1.7531, "ttft_ms": 163.81236553716008, "ttft_relative": 2.0333, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15938.38845416382, "interactivity_tok_s": 64.75765345222887, "output_per_gpu": 513.4772579759764, "total_per_gpu": null, "tpot_error_pct": 6.89, "tpot_ms": 15.44219017660501, "tpot_relative": 1.942483, "ttft_error_pct": 5.88, "ttft_ms": 141.02790349689465, "ttft_relative": 1.75049, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14622.724123553135, "interactivity_tok_s": 63.39846695565878, "output_per_gpu": 489.90591748789416, "total_per_gpu": 980.0327224141982, "tpot_error_pct": 4.89, "tpot_ms": 15.77325206774172, "tpot_relative": 1.984127, "ttft_error_pct": 38.84, "ttft_ms": 91.63970890468734, "ttft_relative": 1.137466, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 15429.899063441553, "interactivity_tok_s": 60.29556166995477, "output_per_gpu": 464.2598723117676, "total_per_gpu": 928.7290688433309, "tpot_ms": 16.584968649496787, "tpot_relative": 2.086233, "ttft_ms": 149.84164435227285, "ttft_relative": 1.85989, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19706.934592931288, "interactivity_tok_s": 52.31796144645334, "output_per_gpu": 830.5705751858061, "total_per_gpu": null, "tpot_error_pct": 11.52, "tpot_ms": 19.11389458519872, "tpot_relative": 2.404349, "ttft_error_pct": 34.63, "ttft_ms": 153.42043227299786, "ttft_relative": 1.904311, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18553.999608066428, "interactivity_tok_s": 49.947075467142945, "output_per_gpu": 771.6081503380867, "total_per_gpu": 1544.911399682479, "tpot_error_pct": 7.32, "tpot_ms": 20.021192244936092, "tpot_relative": 2.518478, "ttft_error_pct": 38.39, "ttft_ms": 144.60677745859422, "ttft_relative": 1.794913, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 20098.074717515556, "interactivity_tok_s": 46.289645591031096, "output_per_gpu": 713.3811764891085, "total_per_gpu": 1428.3295366877903, "tpot_ms": 21.603103398868022, "tpot_relative": 2.717468, "ttft_ms": 234.70524423210009, "ttft_relative": 2.913248, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 118}, {"aic": {"coverage_pct": 58.49, "points": 62, "status_counts": {"failed": 44, "success": 62, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.83, "tpot_shape_error_pct": 8.12, "ttft_mape_pct": 38.47, "ttft_shape_error_pct": 41.79}, "aisimulate": {"coverage_pct": 100.0, "points": 106, "status_counts": {"failed": 0, "success": 106, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.07, "tpot_shape_error_pct": 11.93, "ttft_mape_pct": 27.39, "ttft_shape_error_pct": 13.13}, "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 54.17, "points": 13, "status_counts": {"failed": 11, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.48, "tpot_shape_error_pct": 3.52, "ttft_mape_pct": 27.96, "ttft_shape_error_pct": 40.95}, "aisimulate": {"coverage_pct": 100.0, "points": 24, "status_counts": {"failed": 0, "success": 24, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.33, "tpot_shape_error_pct": 3.18, "ttft_mape_pct": 19.63, "ttft_shape_error_pct": 10.24}, "gpu": "b200", "precisions": ["fp8"], "rows": 24, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.77, "tpot_shape_error_pct": 0.39, "ttft_mape_pct": 2.16, "ttft_shape_error_pct": 4.25}, "framework": "dynamo-vllm", "id": "1a119ba4ec6a58a5", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 45232.971270345304, "interactivity_tok_s": 34.82509988456083, "output_per_gpu": 1433.5314437554705, "total_per_gpu": 12899.137033731573, "tpot_error_pct": 9.59, "tpot_ms": 28.714921229654095, "tpot_relative": 0.904087, "ttft_error_pct": 1.47, "ttft_ms": 18791.616418109996, "ttft_relative": 1.01469, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 47764.154612013226, "interactivity_tok_s": 31.48491403630589, "output_per_gpu": 2375.8411046487668, "total_per_gpu": 12221.358967349839, "tpot_ms": 31.761242824003894, "tpot_relative": 1.0, "ttft_ms": 18519.568818435848, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 87897.29601480805, "interactivity_tok_s": 34.55349073771501, "output_per_gpu": 1468.035459171475, "total_per_gpu": 13200.328002259614, "tpot_error_pct": 9.95, "tpot_ms": 28.9406360587616, "tpot_relative": 0.911193, "ttft_error_pct": 2.84, "ttft_ms": 61229.69686481355, "ttft_relative": 3.306216, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 92633.7639144606, "interactivity_tok_s": 31.116710989480172, "output_per_gpu": 2433.188701466598, "total_per_gpu": 12508.644003656458, "tpot_ms": 32.13707259543197, "tpot_relative": 1.011833, "ttft_ms": 63020.83961206968, "ttft_relative": 3.402932, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 16.67, "points": 1, "status_counts": {"failed": 5, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.11, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.54, "ttft_shape_error_pct": null}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.8, "tpot_shape_error_pct": 5.51, "ttft_mape_pct": 17.84, "ttft_shape_error_pct": 8.2}, "framework": "dynamo-vllm", "id": "88c0be3163c2b43b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12588.816, "interactivity_tok_s": 85.23695874531197, "output_per_gpu": 209.2988333333333, "total_per_gpu": null, "tpot_error_pct": 15.11, "tpot_ms": 11.732, "tpot_relative": 1.151116, "ttft_error_pct": 0.54, "ttft_ms": 586.98, "ttft_relative": 0.994572, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10436.578118900015, "interactivity_tok_s": 91.61607818172854, "output_per_gpu": 225.62095172714908, "total_per_gpu": 2036.1726170724803, "tpot_error_pct": 7.1, "tpot_ms": 10.915114681250731, "tpot_relative": 1.070965, "ttft_error_pct": 22.41, "ttft_ms": 457.93885903124846, "ttft_relative": 0.775926, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 9900.967806700646, "interactivity_tok_s": 98.11759970430755, "output_per_gpu": 358.64829215426704, "total_per_gpu": 2159.226329324323, "tpot_ms": 10.191851441674618, "tpot_relative": 1.0, "ttft_ms": 590.1836930344871, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12397.992366493745, "interactivity_tok_s": 78.64280092586611, "output_per_gpu": 384.2833861494389, "total_per_gpu": 3436.384397462965, "tpot_error_pct": 3.12, "tpot_ms": 12.71572207788817, "tpot_relative": 1.247636, "ttft_error_pct": 19.4, "ttft_ms": 644.692501059374, "ttft_relative": 1.092359, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 12931.869669078878, "interactivity_tok_s": 76.19117110052456, "output_per_gpu": 556.2336882144168, "total_per_gpu": 3318.8015923117255, "tpot_ms": 13.12488029197802, "tpot_relative": 1.287782, "ttft_ms": 799.9040786566184, "ttft_relative": 1.355348, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16216.221482765724, "interactivity_tok_s": 60.07130505118636, "output_per_gpu": 587.5978453425629, "total_per_gpu": 5296.476517046284, "tpot_error_pct": 3.82, "tpot_ms": 16.646883218999598, "tpot_relative": 1.633352, "ttft_error_pct": 22.61, "ttft_ms": 891.3134861000011, "ttft_relative": 1.510231, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 15910.219353209868, "interactivity_tok_s": 62.36765207493304, "output_per_gpu": 895.9084978497822, "total_per_gpu": 5388.018309099987, "tpot_ms": 16.033952966491782, "tpot_relative": 1.573213, "ttft_ms": 1151.6650509984174, "ttft_relative": 1.951367, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20623.4750725867, "interactivity_tok_s": 48.21583084302649, "output_per_gpu": 918.8547845755347, "total_per_gpu": 8295.49619897153, "tpot_error_pct": 3.12, "tpot_ms": 20.74007608114527, "tpot_relative": 2.034966, "ttft_error_pct": 24.42, "ttft_ms": 1563.7723094187484, "ttft_relative": 2.649637, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 20553.98273082592, "interactivity_tok_s": 49.720365260401444, "output_per_gpu": 1385.2031639499605, "total_per_gpu": 8340.938795886534, "tpot_ms": 20.112482978809194, "tpot_relative": 1.973389, "ttft_ms": 2068.9950888328895, "ttft_relative": 3.50568, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 35013.812416171495, "interactivity_tok_s": 45.531147839030524, "output_per_gpu": 1080.6808112064411, "total_per_gpu": 9723.330890339126, "tpot_error_pct": 3.9, "tpot_ms": 21.962986822457683, "tpot_relative": 2.154956, "ttft_error_pct": 12.69, "ttft_ms": 14785.551270406486, "ttft_relative": 25.052456, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 36403.127888794916, "interactivity_tok_s": 47.30611386162007, "output_per_gpu": 1558.85181372921, "total_per_gpu": 9354.293530380231, "tpot_ms": 21.138916693203797, "tpot_relative": 2.0741, "ttft_ms": 16934.315052280363, "ttft_relative": 28.693295, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 68061.21281856844, "interactivity_tok_s": 45.29628150793731, "output_per_gpu": 1105.4845595525042, "total_per_gpu": 9949.441348411017, "tpot_error_pct": 1.76, "tpot_ms": 22.076867387552973, "tpot_relative": 2.166129, "ttft_error_pct": 5.54, "ttft_ms": 47726.25171554301, "ttft_relative": 80.866775, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26849272767", "measured": {"e2e_ms": 71224.0679409827, "interactivity_tok_s": 44.49692711849492, "output_per_gpu": 1586.8145206453266, "total_per_gpu": 9526.400420244592, "tpot_ms": 22.47346198394799, "tpot_relative": 2.205042, "ttft_ms": 50525.24184355261, "ttft_relative": 85.609349, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 87.5, "points": 7, "status_counts": {"failed": 1, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.74, "tpot_shape_error_pct": 4.93, "ttft_mape_pct": 36.29, "ttft_shape_error_pct": 47.22}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.02, "tpot_shape_error_pct": 3.96, "ttft_mape_pct": 25.16, "ttft_shape_error_pct": 11.46}, "framework": "vllm", "id": "1d6ce7f735e14c9d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9166.876162906958, "interactivity_tok_s": 114.01154593168769, "output_per_gpu": 111.59744953678866, "total_per_gpu": null, "tpot_error_pct": 15.56, "tpot_ms": 8.771041492579796, "tpot_relative": 1.15559, "ttft_error_pct": 14.13, "ttft_ms": 194.10071599782665, "ttft_relative": 1.141261, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8058.187226475004, "interactivity_tok_s": 115.76028296264546, "output_per_gpu": 111.43621341820736, "total_per_gpu": 1001.7450995919303, "tpot_error_pct": 13.81, "tpot_ms": 8.638541427224126, "tpot_relative": 1.138133, "ttft_error_pct": 30.28, "ttft_ms": 118.57094052499943, "ttft_relative": 0.697166, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 7145.431127701886, "interactivity_tok_s": 131.75054710015914, "output_per_gpu": 125.36493367576148, "total_per_gpu": 1126.9559878085604, "tpot_ms": 7.590101308951544, "tpot_relative": 1.0, "ttft_ms": 170.07568636909127, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10611.042241895353, "interactivity_tok_s": 98.3911877855529, "output_per_gpu": 192.818005371972, "total_per_gpu": null, "tpot_error_pct": 15.79, "tpot_ms": 10.163511819570017, "tpot_relative": 1.339048, "ttft_error_pct": 0.65, "ttft_ms": 213.7696504752252, "ttft_relative": 1.256909, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9434.987462474997, "interactivity_tok_s": 99.56746160769781, "output_per_gpu": 192.82941992441113, "total_per_gpu": 1715.3068428216416, "tpot_error_pct": 14.43, "tpot_ms": 10.043441741440233, "tpot_relative": 1.323229, "ttft_error_pct": 33.95, "ttft_ms": 140.2829212374997, "ttft_relative": 0.824826, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 8335.93658575337, "interactivity_tok_s": 113.93051190874873, "output_per_gpu": 217.77757596109316, "total_per_gpu": 1937.232225277689, "tpot_ms": 8.777279968696515, "tpot_relative": 1.156411, "ttft_ms": 212.3890184368065, "ttft_relative": 1.248791, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12434.696363191208, "interactivity_tok_s": 83.84367143084403, "output_per_gpu": 329.0792055134541, "total_per_gpu": null, "tpot_error_pct": 9.83, "tpot_ms": 11.926958623523786, "tpot_relative": 1.571383, "ttft_error_pct": 11.74, "ttft_ms": 233.4176913263751, "ttft_relative": 1.372434, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10963.375341275036, "interactivity_tok_s": 84.84909038608721, "output_per_gpu": 323.06765409396974, "total_per_gpu": 2915.604715308136, "tpot_error_pct": 8.53, "tpot_ms": 11.785630175287901, "tpot_relative": 1.552763, "ttft_error_pct": 30.94, "ttft_ms": 182.63195384374993, "ttft_relative": 1.073828, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 10197.931063637225, "interactivity_tok_s": 92.08842188579091, "output_per_gpu": 347.90700667587816, "total_per_gpu": 3139.7736551424787, "tpot_ms": 10.859128428112399, "tpot_relative": 1.430696, "ttft_ms": 264.4711139902938, "ttft_relative": 1.55502, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16185.37134896647, "interactivity_tok_s": 64.21550527919332, "output_per_gpu": 505.64178130658684, "total_per_gpu": null, "tpot_error_pct": 6.6, "tpot_ms": 15.572562976063871, "tpot_relative": 2.051694, "ttft_error_pct": 33.22, "ttft_ms": 254.6394244531294, "ttft_relative": 1.497212, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14452.76213617503, "interactivity_tok_s": 65.16949742419675, "output_per_gpu": 498.00995361440283, "total_per_gpu": 4453.363575068226, "tpot_error_pct": 5.04, "tpot_ms": 15.344601992107897, "tpot_relative": 2.02166, "ttft_error_pct": 33.41, "ttft_ms": 253.94165860937497, "ttft_relative": 1.49311, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 13900.724799081217, "interactivity_tok_s": 68.451370060662, "output_per_gpu": 518.4209081598788, "total_per_gpu": 4635.884829604387, "tpot_ms": 14.608911393793788, "tpot_relative": 1.924732, "ttft_ms": 381.3250686536776, "ttft_relative": 2.24209, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23229.655736598772, "interactivity_tok_s": 44.56979416868046, "output_per_gpu": 704.6165550448472, "total_per_gpu": null, "tpot_error_pct": 8.71, "tpot_ms": 22.436720174550587, "tpot_relative": 2.95605, "ttft_error_pct": 51.45, "ttft_ms": 276.89099803352127, "ttft_relative": 1.628046, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21013.371340596863, "interactivity_tok_s": 44.72644564330558, "output_per_gpu": 688.2262803464795, "total_per_gpu": 6203.51888823579, "tpot_error_pct": 8.33, "tpot_ms": 22.358137017527007, "tpot_relative": 2.945697, "ttft_error_pct": 29.11, "ttft_ms": 404.2810773765626, "ttft_relative": 2.377066, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 19595.908435009915, "interactivity_tok_s": 48.45052383114646, "output_per_gpu": 737.4795851429484, "total_per_gpu": 6647.47724225143, "tpot_ms": 20.639611730206912, "tpot_relative": 2.71928, "ttft_ms": 570.2681942770141, "ttft_relative": 3.353026, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33746.59788333081, "interactivity_tok_s": 30.589275741121252, "output_per_gpu": 970.0533402855997, "total_per_gpu": null, "tpot_error_pct": 9.25, "tpot_ms": 32.691195713918034, "tpot_relative": 4.307083, "ttft_error_pct": 64.6, "ttft_ms": 303.5046679926566, "ttft_relative": 1.784527, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30764.2453380746, "interactivity_tok_s": 30.621046513106172, "output_per_gpu": 940.6566660709832, "total_per_gpu": 8492.325369491078, "tpot_error_pct": 9.13, "tpot_ms": 32.65727706504048, "tpot_relative": 4.302614, "ttft_error_pct": 17.81, "ttft_ms": 704.7754830179672, "ttft_relative": 4.143893, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 28406.0171119932, "interactivity_tok_s": 33.41796176350437, "output_per_gpu": 1019.2000660175104, "total_per_gpu": 9201.421612606038, "tpot_ms": 29.924027296365395, "tpot_relative": 3.942507, "ttft_ms": 857.4749151280685, "ttft_relative": 5.041725, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 53965.6970288364, "interactivity_tok_s": 19.07552096005781, "output_per_gpu": 1213.2151274728321, "total_per_gpu": null, "tpot_error_pct": 9.43, "tpot_ms": 52.42320784286299, "tpot_relative": 6.906786, "ttft_error_pct": 78.23, "ttft_ms": 336.7554055875599, "ttft_relative": 1.980033, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 49597.91475945586, "interactivity_tok_s": 19.115230373192258, "output_per_gpu": 1174.2928751988409, "total_per_gpu": 10565.597232155225, "tpot_error_pct": 9.21, "tpot_ms": 52.31430542435044, "tpot_relative": 6.892438, "ttft_error_pct": 14.33, "ttft_ms": 1325.2738254792962, "ttft_relative": 7.792259, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 45749.503230036265, "interactivity_tok_s": 20.87527560497547, "output_per_gpu": 1272.9250937083343, "total_per_gpu": 11453.031974283567, "tpot_ms": 47.90355916362883, "tpot_relative": 6.31132, "ttft_ms": 1547.0411867751864, "ttft_relative": 9.096193, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 91596.8680111135, "interactivity_tok_s": 11.880290339279101, "output_per_gpu": 1268.0807548935395, "total_per_gpu": 11412.818918944675, "tpot_error_pct": 11.71, "tpot_ms": 84.17302704242498, "tpot_relative": 11.089842, "ttft_error_pct": 11.43, "ttft_ms": 13883.390441136606, "ttft_relative": 81.630659, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 85246.07949034798, "interactivity_tok_s": 13.27177804253469, "output_per_gpu": 1365.0129122639787, "total_per_gpu": 12285.215377310882, "tpot_ms": 75.34785443179523, "tpot_relative": 9.927121, "ttft_ms": 15674.458468685225, "ttft_relative": 92.161665, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 62.5, "points": 5, "status_counts": {"failed": 3, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.79, "tpot_shape_error_pct": 1.41, "ttft_mape_pct": 21.78, "ttft_shape_error_pct": 31.54}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.67, "tpot_shape_error_pct": 1.13, "ttft_mape_pct": 19.81, "ttft_shape_error_pct": 11.35}, "framework": "vllm", "id": "10e2272bfb254487", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 10510.355948048951, "interactivity_tok_s": 99.85416975702677, "output_per_gpu": 194.6651483653892, "total_per_gpu": null, "tpot_error_pct": 12.05, "tpot_ms": 10.01460432181531, "tpot_relative": 1.120514, "ttft_error_pct": 9.98, "ttft_ms": 265.41572683188855, "ttft_relative": 1.099777, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9201.185677000027, "interactivity_tok_s": 101.84725161445043, "output_per_gpu": 195.30566258963393, "total_per_gpu": 1755.6814290475024, "tpot_error_pct": 9.86, "tpot_ms": 9.818625285889567, "tpot_relative": 1.098586, "ttft_error_pct": 26.66, "ttft_ms": 177.00050775000068, "ttft_relative": 0.733419, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 8455.128846570733, "interactivity_tok_s": 111.88794844626545, "output_per_gpu": 212.7385081795555, "total_per_gpu": 1912.3923141895623, "tpot_ms": 8.937513055575, "tpot_relative": 1.0, "ttft_ms": 241.33597301406553, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13023.409694533057, "interactivity_tok_s": 80.35376767494232, "output_per_gpu": 314.2034302827571, "total_per_gpu": null, "tpot_error_pct": 14.55, "tpot_ms": 12.444967161282744, "tpot_relative": 1.392442, "ttft_error_pct": 0.05, "ttft_ms": 292.20828854080975, "ttft_relative": 1.210795, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11457.184615675036, "interactivity_tok_s": 82.30105485288522, "output_per_gpu": 317.9995001624179, "total_per_gpu": 2828.752577569747, "tpot_error_pct": 11.84, "tpot_ms": 12.150512551601192, "tpot_relative": 1.359496, "ttft_error_pct": 28.2, "ttft_ms": 209.7086720625004, "ttft_relative": 0.868949, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 10350.681547197746, "interactivity_tok_s": 92.04768097591102, "output_per_gpu": 351.995751460809, "total_per_gpu": 3131.164950667552, "tpot_ms": 10.863934749879261, "tpot_relative": 1.215543, "ttft_ms": 292.07506117527373, "ttft_relative": 1.210243, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17083.48964454596, "interactivity_tok_s": 61.02647047046578, "output_per_gpu": 479.0590312801113, "total_per_gpu": null, "tpot_error_pct": 11.61, "tpot_ms": 16.386331903038002, "tpot_relative": 1.833433, "ttft_error_pct": 14.22, "ttft_ms": 320.2721077380865, "ttft_relative": 1.32708, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14899.830144218742, "interactivity_tok_s": 62.557970796219756, "output_per_gpu": 477.9744785471759, "total_per_gpu": 4313.600033272738, "tpot_error_pct": 8.88, "tpot_ms": 15.985173228483745, "tpot_relative": 1.788548, "ttft_error_pct": 25.96, "ttft_ms": 276.42372284375006, "ttft_relative": 1.14539, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 13805.864378363185, "interactivity_tok_s": 68.11046869691175, "output_per_gpu": 515.1362992678509, "total_per_gpu": 4648.9761637816755, "tpot_ms": 14.682030811591547, "tpot_relative": 1.642742, "ttft_ms": 373.3562226927461, "ttft_relative": 1.547039, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23097.097357810697, "interactivity_tok_s": 44.971535277437255, "output_per_gpu": 708.6604756621017, "total_per_gpu": null, "tpot_error_pct": 9.01, "tpot_ms": 22.236287772494876, "tpot_relative": 2.487973, "ttft_error_pct": 33.22, "ttft_ms": 349.37496654843846, "ttft_relative": 1.44767, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20500.077362284435, "interactivity_tok_s": 46.00857033826086, "output_per_gpu": 705.018757317894, "total_per_gpu": 6304.502210834068, "tpot_error_pct": 6.55, "tpot_ms": 21.73508093487524, "tpot_relative": 2.431894, "ttft_error_pct": 26.0, "ttft_ms": 387.19909659062506, "ttft_relative": 1.604399, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 19398.26313629892, "interactivity_tok_s": 49.023863507286826, "output_per_gpu": 744.8144283653949, "total_per_gpu": 6660.3677725604875, "tpot_ms": 20.398229116547732, "tpot_relative": 2.282316, "ttft_ms": 523.2107575939153, "ttft_relative": 2.167977, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34762.682018200016, "interactivity_tok_s": 29.755385341048715, "output_per_gpu": 941.6994920835238, "total_per_gpu": null, "tpot_error_pct": 11.72, "tpot_ms": 33.607361777985815, "tpot_relative": 3.760259, "ttft_error_pct": 51.44, "ttft_ms": 382.35091932052705, "ttft_relative": 1.58431, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31089.417572581162, "interactivity_tok_s": 30.252967020647453, "output_per_gpu": 932.4648190034118, "total_per_gpu": 8405.0308488523, "tpot_error_pct": 9.89, "tpot_ms": 33.05460913362668, "tpot_relative": 3.698412, "ttft_error_pct": 21.26, "ttft_ms": 619.9415736796884, "ttft_relative": 2.568791, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 28520.05264639083, "interactivity_tok_s": 33.24383348180115, "output_per_gpu": 1016.0290557238234, "total_per_gpu": 9158.260325377232, "tpot_ms": 30.080766724674984, "tpot_relative": 3.365675, "ttft_ms": 787.3383872421982, "ttft_relative": 3.262416, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 51517.030364574974, "interactivity_tok_s": 21.837971189008023, "output_per_gpu": 1114.2868495353653, "total_per_gpu": 10059.87287659894, "tpot_error_pct": 9.23, "tpot_ms": 45.79179958362352, "tpot_relative": 5.123551, "ttft_error_pct": 25.71, "ttft_ms": 9348.734722317973, "ttft_relative": 38.737427, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 51172.72076758709, "interactivity_tok_s": 23.853080860796325, "output_per_gpu": 1120.5067095545937, "total_per_gpu": 10116.02628191802, "tpot_ms": 41.92330566587512, "tpot_relative": 4.690713, "ttft_ms": 12583.362595703147, "ttft_relative": 52.140435, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 101457.89422019426, "interactivity_tok_s": 21.692199587179278, "output_per_gpu": 1124.8484405255795, "total_per_gpu": 10120.725264384124, "tpot_error_pct": 9.68, "tpot_ms": 46.09952052031778, "tpot_relative": 5.157981, "ttft_error_pct": 4.47, "ttft_ms": 58916.52272540793, "ttft_relative": 244.126568, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 100452.48859084805, "interactivity_tok_s": 23.793022861836235, "output_per_gpu": 1133.3741367475404, "total_per_gpu": 10197.434468967942, "tpot_ms": 42.029127858486184, "tpot_relative": 4.702553, "ttft_ms": 61673.01066738837, "ttft_relative": 255.548354, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 201093.03171334916, "interactivity_tok_s": 21.55892961968554, "output_per_gpu": 1126.7396677079926, "total_per_gpu": 10140.73886597451, "tpot_error_pct": 11.42, "tpot_ms": 46.38449207083529, "tpot_relative": 5.189866, "ttft_error_pct": 0.19, "ttft_ms": 158307.26605205116, "ttft_relative": 655.96216, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696233215", "measured": {"e2e_ms": 196417.3185408579, "interactivity_tok_s": 24.021061903157108, "output_per_gpu": 1154.219854886341, "total_per_gpu": 10388.062546990019, "tpot_ms": 41.630132923830864, "tpot_relative": 4.65791, "ttft_ms": 158008.96119226154, "ttft_relative": 654.726103, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 73.08, "points": 19, "status_counts": {"failed": 7, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.87, "tpot_shape_error_pct": 20.89, "ttft_mape_pct": 51.22, "ttft_shape_error_pct": 52.31}, "aisimulate": {"coverage_pct": 100.0, "points": 26, "status_counts": {"failed": 0, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.89, "tpot_shape_error_pct": 37.69, "ttft_mape_pct": 34.67, "ttft_shape_error_pct": 20.88}, "gpu": "b300", "precisions": ["fp8"], "rows": 26, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.78, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.92, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 38.75, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "2baa053cee571f41", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 22145.221, "interactivity_tok_s": 47.19875395289564, "output_per_gpu": 347.6275, "total_per_gpu": null, "tpot_error_pct": 18.78, "tpot_ms": 21.187, "tpot_relative": 1.187843, "ttft_error_pct": 31.92, "ttft_ms": 470.92, "ttft_relative": 0.680827, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17911.195523656173, "interactivity_tok_s": 52.85145101392697, "output_per_gpu": 401.4417692819078, "total_per_gpu": 3589.8201227825566, "tpot_error_pct": 6.08, "tpot_ms": 18.920956394110892, "tpot_relative": 1.060798, "ttft_error_pct": 38.75, "ttft_ms": 423.64559667187376, "ttft_relative": 0.61248, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 18938.459694034238, "interactivity_tok_s": 56.06471337248735, "output_per_gpu": 857.0562033293869, "total_per_gpu": 3856.752914982241, "tpot_ms": 17.836531034344507, "tpot_relative": 1.0, "ttft_ms": 691.6884458998084, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 75.0, "points": 3, "status_counts": {"failed": 1, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 50.12, "tpot_shape_error_pct": 19.71, "ttft_mape_pct": 68.43, "ttft_shape_error_pct": 63.03}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 47.31, "tpot_shape_error_pct": 32.85, "ttft_mape_pct": 42.95, "ttft_shape_error_pct": 28.95}, "framework": "dynamo-vllm", "id": "f2b8281fdc13d692", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 21980.518, "interactivity_tok_s": 47.56016360696281, "output_per_gpu": 467.11466666666666, "total_per_gpu": null, "tpot_error_pct": 32.68, "tpot_ms": 21.026, "tpot_relative": 1.326796, "ttft_error_pct": 45.56, "ttft_ms": 470.92, "ttft_relative": 0.544421, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17862.882066164035, "interactivity_tok_s": 53.39105581634065, "output_per_gpu": 535.2703638314574, "total_per_gpu": 4824.80821666617, "tpot_error_pct": 18.19, "tpot_ms": 18.729728878932264, "tpot_relative": 1.181895, "ttft_error_pct": 28.24, "ttft_ms": 620.7461704687497, "ttft_relative": 0.717632, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 15451.699416254678, "interactivity_tok_s": 63.10264007298505, "output_per_gpu": 926.5724571048459, "total_per_gpu": 5572.432202139633, "tpot_ms": 15.847197499873088, "tpot_relative": 1.0, "ttft_ms": 864.9920186014242, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29143.564, "interactivity_tok_s": 35.678607107178536, "output_per_gpu": 700.9074999999999, "total_per_gpu": null, "tpot_error_pct": 39.93, "tpot_ms": 28.028, "tpot_relative": 1.768641, "ttft_error_pct": 64.9, "ttft_ms": 470.92, "ttft_relative": 0.544421, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23718.161626951576, "interactivity_tok_s": 40.42521228270187, "output_per_gpu": 802.4708812872532, "total_per_gpu": 7244.772794624884, "tpot_error_pct": 23.5, "tpot_ms": 24.737037693377914, "tpot_relative": 1.560972, "ttft_error_pct": 26.51, "ttft_ms": 985.89084869453, "ttft_relative": 1.139769, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 19749.19625206953, "interactivity_tok_s": 49.92467150195389, "output_per_gpu": 1439.4222426023268, "total_per_gpu": 8667.416549026493, "tpot_ms": 20.030176862773413, "tpot_relative": 1.263957, "ttft_ms": 1341.5993656687306, "ttft_relative": 1.550996, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 43335.643, "interactivity_tok_s": 23.865778859693084, "output_per_gpu": 937.6836666666667, "total_per_gpu": null, "tpot_error_pct": 77.74, "tpot_ms": 41.901, "tpot_relative": 2.644064, "ttft_error_pct": 94.85, "ttft_ms": 470.92, "ttft_relative": 0.544421, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34689.119405278456, "interactivity_tok_s": 28.063035472428705, "output_per_gpu": 1101.200955719659, "total_per_gpu": 9907.959092256446, "tpot_error_pct": 51.16, "tpot_ms": 35.63406392663678, "tpot_relative": 2.248604, "ttft_error_pct": 79.54, "ttft_ms": 1870.2093946535138, "ttft_relative": 2.162112, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 30849.70414117303, "interactivity_tok_s": 42.418786467794284, "output_per_gpu": 1841.075277343122, "total_per_gpu": 11047.848425434364, "tpot_ms": 23.574460357540694, "tpot_relative": 1.487611, "ttft_ms": 9138.89416139845, "ttft_relative": 10.565293, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 66815.1261888613, "interactivity_tok_s": 21.287666446425657, "output_per_gpu": 1134.0800182848905, "total_per_gpu": 10206.802554436324, "tpot_error_pct": 96.4, "tpot_ms": 46.975557537820535, "tpot_relative": 2.964282, "ttft_error_pct": 37.53, "ttft_ms": 23612.13424055714, "ttft_relative": 27.297517, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 59827.26142086325, "interactivity_tok_s": 41.809776101910394, "output_per_gpu": 1885.2453793359336, "total_per_gpu": 11318.022453352767, "tpot_ms": 23.917851116029, "tpot_relative": 1.50928, "ttft_ms": 37797.924546621834, "ttft_relative": 43.697426, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 33.33, "points": 1, "status_counts": {"failed": 2, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.78, "tpot_shape_error_pct": null, "ttft_mape_pct": 76.33, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 101.9, "tpot_shape_error_pct": 88.09, "ttft_mape_pct": 54.68, "ttft_shape_error_pct": 49.39}, "framework": "dynamo-vllm", "id": "7920c1e99ab7c66b", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 41459.461, "interactivity_tok_s": 24.95819502333591, "output_per_gpu": 905.2663846153846, "total_per_gpu": null, "tpot_error_pct": 43.78, "tpot_ms": 40.067, "tpot_relative": 1.437766, "ttft_error_pct": 76.33, "ttft_ms": 470.92, "ttft_relative": 0.236655, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33994.09546807331, "interactivity_tok_s": 28.209821828688764, "output_per_gpu": 1036.2314820514714, "total_per_gpu": 9326.158619729214, "tpot_error_pct": 27.2, "tpot_ms": 35.44864643501655, "tpot_relative": 1.272041, "ttft_error_pct": 32.43, "ttft_ms": 1344.5467556658175, "ttft_relative": 0.675686, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 27654.726834746703, "interactivity_tok_s": 35.884046304103535, "output_per_gpu": 2061.2746561052186, "total_per_gpu": 11422.901277813533, "tpot_ms": 27.86753733192136, "tpot_relative": 1.0, "ttft_ms": 1989.8991032525203, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 56563.95159301977, "interactivity_tok_s": 17.943530281999614, "output_per_gpu": 1238.6114525318203, "total_per_gpu": 11145.216888931103, "tpot_error_pct": 87.11, "tpot_ms": 55.730393310795066, "tpot_relative": 1.999832, "ttft_error_pct": 75.44, "ttft_ms": 5295.472442309241, "ttft_relative": 2.661176, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 48984.28551773164, "interactivity_tok_s": 33.57407604856325, "output_per_gpu": 2306.7296681574167, "total_per_gpu": 12778.606397552428, "tpot_ms": 29.784885176096854, "tpot_relative": 1.068802, "ttft_ms": 21559.738439193676, "ttft_relative": 10.834589, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 110514.99812493866, "interactivity_tok_s": 11.362519840518964, "output_per_gpu": 1257.9367855423993, "total_per_gpu": 11311.156056570566, "tpot_error_pct": 191.4, "tpot_ms": 88.00864720464389, "tpot_relative": 3.158106, "ttft_error_pct": 56.17, "ttft_ms": 29607.842399019108, "ttft_relative": 14.879067, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 95388.87244671759, "interactivity_tok_s": 33.110321629059754, "output_per_gpu": 2360.3045730966246, "total_per_gpu": 13067.339444542087, "tpot_ms": 30.2020624022672, "tpot_relative": 1.083772, "ttft_ms": 67558.8886745875, "ttft_relative": 33.950912, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 25.0, "points": 1, "status_counts": {"failed": 3, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.77, "tpot_shape_error_pct": null, "ttft_mape_pct": 71.5, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 101.68, "tpot_shape_error_pct": 89.77, "ttft_mape_pct": 34.74, "ttft_shape_error_pct": 20.0}, "framework": "dynamo-vllm", "id": "3e6ad04c57b6d0c9", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 41459.461, "interactivity_tok_s": 24.95819502333591, "output_per_gpu": 840.6045714285714, "total_per_gpu": null, "tpot_error_pct": 35.77, "tpot_ms": 40.067, "tpot_relative": 1.357703, "ttft_error_pct": 71.5, "ttft_ms": 470.92, "ttft_relative": 0.284998, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33925.772432107035, "interactivity_tok_s": 28.113481775531195, "output_per_gpu": 965.3236346273487, "total_per_gpu": 8685.414803718128, "tpot_error_pct": 20.53, "tpot_ms": 35.57012283232589, "tpot_relative": 1.205323, "ttft_error_pct": 29.45, "ttft_ms": 1165.6891328027368, "ttft_relative": 0.705467, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 31841.98143108464, "interactivity_tok_s": 33.88582251567661, "output_per_gpu": 1998.4324774580216, "total_per_gpu": 10277.652741212683, "tpot_ms": 29.510866957335022, "tpot_relative": 1.0, "ttft_ms": 1652.364533730912, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 56372.34643082941, "interactivity_tok_s": 17.73630717406477, "output_per_gpu": 1154.374786894243, "total_per_gpu": 10389.456946316805, "tpot_error_pct": 44.26, "tpot_ms": 56.38152238715553, "tpot_relative": 1.910534, "ttft_error_pct": 35.18, "ttft_ms": 4392.780709271463, "ttft_relative": 2.658482, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 46759.791583861625, "interactivity_tok_s": 25.585994542820213, "output_per_gpu": 2721.173370485824, "total_per_gpu": 13994.605905355666, "tpot_ms": 39.083882329702675, "tpot_relative": 1.324389, "ttft_ms": 6776.979960575787, "ttft_relative": 4.101383, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 110133.16755040977, "interactivity_tok_s": 10.40625574684035, "output_per_gpu": 1173.7682272780296, "total_per_gpu": 10561.747546907765, "tpot_error_pct": 144.19, "tpot_ms": 96.09604302715988, "tpot_relative": 3.256293, "ttft_error_pct": 55.46, "ttft_ms": 21815.799552078082, "ttft_relative": 13.202776, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 89239.85127065874, "interactivity_tok_s": 25.41058759170972, "output_per_gpu": 2818.39744782674, "total_per_gpu": 14494.615445966092, "tpot_ms": 39.35367477792025, "tpot_relative": 1.333532, "ttft_ms": 48981.04197284632, "ttft_relative": 29.643, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 217678.2882509808, "interactivity_tok_s": 8.440767801826457, "output_per_gpu": 1182.6647316940378, "total_per_gpu": 10634.322405179817, "tpot_error_pct": 197.75, "tpot_ms": 118.47263465577325, "tpot_relative": 4.014543, "ttft_error_pct": 18.87, "ttft_ms": 108849.61158194195, "ttft_relative": 65.875059, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 174868.35931932688, "interactivity_tok_s": 25.132579896915416, "output_per_gpu": 2859.008574341369, "total_per_gpu": 14703.472668041326, "tpot_ms": 39.78899118600763, "tpot_relative": 1.348283, "ttft_ms": 134164.22133604108, "ttft_relative": 81.195292, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 7}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.96, "tpot_shape_error_pct": null, "ttft_mape_pct": 32.81, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.18, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.25, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "d795496ad66def9c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 21482.317, "interactivity_tok_s": 48.68786211597448, "output_per_gpu": 409.93942857142855, "total_per_gpu": null, "tpot_error_pct": 27.96, "tpot_ms": 20.539, "tpot_relative": 1.279625, "ttft_error_pct": 32.81, "ttft_ms": 470.92, "ttft_relative": 0.671885, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17359.769814232794, "interactivity_tok_s": 54.56698228550216, "output_per_gpu": 472.733617977685, "total_per_gpu": 4261.11587420379, "tpot_error_pct": 14.18, "tpot_ms": 18.326100475336144, "tpot_relative": 1.141757, "ttft_error_pct": 30.25, "ttft_ms": 488.8435829484362, "ttft_relative": 0.697457, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 17120.85756825536, "interactivity_tok_s": 62.30220811004168, "output_per_gpu": 944.2654335653591, "total_per_gpu": 4856.222229764704, "tpot_ms": 16.050795474756583, "tpot_relative": 1.0, "ttft_ms": 700.8937975793742, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 7}, {"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.6, "tpot_shape_error_pct": 51.07, "ttft_mape_pct": 75.33, "ttft_shape_error_pct": 86.67}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.41, "tpot_shape_error_pct": 20.47, "ttft_mape_pct": 38.32, "ttft_shape_error_pct": 19.73}, "framework": "dynamo-vllm", "id": "836ea4aabdae5e57", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13426.192, "interactivity_tok_s": 78.96399241945673, "output_per_gpu": 232.6938, "total_per_gpu": null, "tpot_error_pct": 26.25, "tpot_ms": 12.664, "tpot_relative": 1.262458, "ttft_error_pct": 29.49, "ttft_ms": 470.92, "ttft_relative": 0.705051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11035.436448981265, "interactivity_tok_s": 86.31388508765886, "output_per_gpu": 256.14353549945423, "total_per_gpu": 2311.6312959039906, "tpot_error_pct": 15.5, "tpot_ms": 11.585621467327275, "tpot_relative": 1.154956, "ttft_error_pct": 33.53, "ttft_ms": 443.9605442937508, "ttft_relative": 0.664688, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 9832.047766862343, "interactivity_tok_s": 99.6887276368999, "output_per_gpu": 360.32584295571206, "total_per_gpu": 2603.1911405103224, "tpot_ms": 10.031224429329047, "tpot_relative": 1.0, "ttft_ms": 667.9228944862189, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21980.518, "interactivity_tok_s": 47.56016360696281, "output_per_gpu": 560.5376, "total_per_gpu": null, "tpot_error_pct": 34.27, "tpot_ms": 21.026, "tpot_relative": 2.096055, "ttft_error_pct": 75.89, "ttft_ms": 470.92, "ttft_relative": 0.705051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18217.049163346855, "interactivity_tok_s": 53.82464244578973, "output_per_gpu": 627.097293794761, "total_per_gpu": 5652.515775565656, "tpot_error_pct": 18.64, "tpot_ms": 18.578850774664495, "tpot_relative": 1.852102, "ttft_error_pct": 42.91, "ttft_ms": 1114.9456585531266, "ttft_relative": 1.669273, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 16367.724711152687, "interactivity_tok_s": 63.8576304010766, "output_per_gpu": 869.605281930927, "total_per_gpu": 6275.796055484458, "tpot_ms": 15.659835695737634, "tpot_relative": 1.561109, "ttft_ms": 1953.08336125172, "ttft_relative": 2.924115, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29143.564, "interactivity_tok_s": 35.678607107178536, "output_per_gpu": 704.471, "total_per_gpu": null, "tpot_error_pct": 75.82, "tpot_ms": 28.028, "tpot_relative": 2.794076, "ttft_error_pct": 96.98, "ttft_ms": 470.92, "ttft_relative": 0.705051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27134.18157276718, "interactivity_tok_s": 43.209063562980475, "output_per_gpu": 839.0595238609656, "total_per_gpu": 7575.098054383935, "tpot_error_pct": 45.18, "tpot_ms": 23.143292576623985, "tpot_relative": 2.307125, "ttft_error_pct": 62.34, "ttft_ms": 5866.249444431242, "ttft_relative": 8.782824, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 30228.944513852708, "interactivity_tok_s": 62.730682629271094, "output_per_gpu": 935.4957062454992, "total_per_gpu": 6759.654582963599, "tpot_ms": 15.941162411859118, "tpot_relative": 1.589154, "ttft_ms": 15577.182024612124, "ttft_relative": 23.321827, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 43335.643, "interactivity_tok_s": 23.865778859693084, "output_per_gpu": 704.471, "total_per_gpu": null, "tpot_error_pct": 162.06, "tpot_ms": 41.901, "tpot_relative": 4.177057, "ttft_error_pct": 98.95, "ttft_ms": 470.92, "ttft_relative": 0.705051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52193.534882749605, "interactivity_tok_s": 42.72929537850555, "output_per_gpu": 868.9437390868823, "total_per_gpu": 7818.245140114972, "tpot_error_pct": 46.37, "tpot_ms": 23.40314744583965, "tpot_relative": 2.33303, "ttft_error_pct": 31.42, "ttft_ms": 30638.78414329903, "ttft_relative": 45.871738, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 59401.46484077659, "interactivity_tok_s": 62.54327579545226, "output_per_gpu": 948.9262698277719, "total_per_gpu": 6833.133044575377, "tpot_ms": 15.98892906202258, "tpot_relative": 1.593916, "ttft_ms": 44675.48725032581, "ttft_relative": 66.887193, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 102379.07933606404, "interactivity_tok_s": 42.50384627304264, "output_per_gpu": 881.2184986570397, "total_per_gpu": 7931.0305076283785, "tpot_error_pct": 46.37, "tpot_ms": 23.52728253288064, "tpot_relative": 2.345405, "ttft_error_pct": 21.37, "ttft_ms": 80708.27306894916, "ttft_relative": 120.834716, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26962351977", "measured": {"e2e_ms": 117450.35455718444, "interactivity_tok_s": 62.21191435681551, "output_per_gpu": 957.5931417586683, "total_per_gpu": 6898.663143841065, "tpot_ms": 16.074091439535437, "tpot_relative": 1.602406, "ttft_ms": 102645.78366861914, "ttft_relative": 153.679092, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.61, "tpot_shape_error_pct": 0.86, "ttft_mape_pct": 11.93, "ttft_shape_error_pct": 8.18}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.22, "tpot_shape_error_pct": 1.03, "ttft_mape_pct": 30.45, "ttft_shape_error_pct": 1.62}, "framework": "vllm", "id": "b1e8772da7b2cf63", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8957.617345973826, "interactivity_tok_s": 116.62507601294911, "output_per_gpu": 114.20447653524819, "total_per_gpu": null, "tpot_error_pct": 14.12, "tpot_ms": 8.574485300991084, "tpot_relative": 1.141241, "ttft_error_pct": 16.7, "ttft_ms": 185.9188830599466, "ttft_relative": 1.166993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7889.891995599998, "interactivity_tok_s": 118.16302197709389, "output_per_gpu": 113.75460183091874, "total_per_gpu": 1022.5860287669778, "tpot_error_pct": 12.64, "tpot_ms": 8.462884439379451, "tpot_relative": 1.126387, "ttft_error_pct": 29.88, "ttft_ms": 111.70792132500029, "ttft_relative": 0.701179, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 7065.5617156022345, "interactivity_tok_s": 133.0973065305125, "output_per_gpu": 127.12303348547442, "total_per_gpu": 1142.7602565911327, "tpot_ms": 7.513300051423283, "tpot_relative": 1.0, "ttft_ms": 159.31452027289197, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10374.793495681726, "interactivity_tok_s": 100.58911893704183, "output_per_gpu": 197.20874452600924, "total_per_gpu": null, "tpot_error_pct": 15.1, "tpot_ms": 9.941433134789603, "tpot_relative": 1.323178, "ttft_error_pct": 7.15, "ttft_ms": 204.70739879196137, "ttft_relative": 1.284926, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9227.38408501253, "interactivity_tok_s": 101.74507300934798, "output_per_gpu": 197.11111489166174, "total_per_gpu": 1753.394499150634, "tpot_error_pct": 13.79, "tpot_ms": 9.828485747984313, "tpot_relative": 1.308145, "ttft_error_pct": 31.02, "ttft_ms": 131.7833778875003, "ttft_relative": 0.82719, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 8185.038795418222, "interactivity_tok_s": 115.77911956648882, "output_per_gpu": 221.92513477451712, "total_per_gpu": 1974.1266785019952, "tpot_ms": 8.637135985696688, "tpot_relative": 1.14958, "ttft_ms": 191.0407881368883, "ttft_relative": 1.199142, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.2, "tpot_shape_error_pct": 5.26, "ttft_mape_pct": 65.8, "ttft_shape_error_pct": 44.94}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.6, "tpot_shape_error_pct": 4.7, "ttft_mape_pct": 15.26, "ttft_shape_error_pct": 10.09}, "framework": "vllm", "id": "deec03fb596e7853", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 33499.1672118191, "interactivity_tok_s": 30.861946080636574, "output_per_gpu": 977.2183228617744, "total_per_gpu": null, "tpot_error_pct": 9.02, "tpot_ms": 32.40236365481245, "tpot_relative": 1.090177, "ttft_error_pct": 51.16, "ttft_ms": 351.5491929459689, "ttft_relative": 0.488361, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30153.265293929766, "interactivity_tok_s": 31.158557855933857, "output_per_gpu": 961.119951860445, "total_per_gpu": 8663.32185429606, "tpot_error_pct": 7.98, "tpot_ms": 32.09391155468896, "tpot_relative": 1.079799, "ttft_error_pct": 20.6, "ttft_ms": 571.5594172265625, "ttft_relative": 0.793992, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 28117.712138204115, "interactivity_tok_s": 33.64498739020258, "output_per_gpu": 1030.979227403697, "total_per_gpu": 9293.017853601486, "tpot_ms": 29.72210951968435, "tpot_relative": 1.0, "ttft_ms": 719.855662297141, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48735.863987511795, "interactivity_tok_s": 21.158714693523304, "output_per_gpu": 1343.4049310539917, "total_per_gpu": null, "tpot_error_pct": 3.05, "tpot_ms": 47.26184999819959, "tpot_relative": 1.590124, "ttft_error_pct": 66.7, "ttft_ms": 386.99143935361775, "ttft_relative": 0.537596, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44313.35310451417, "interactivity_tok_s": 21.25041381053353, "output_per_gpu": 1307.8846624446353, "total_per_gpu": 11807.689776589224, "tpot_error_pct": 2.61, "tpot_ms": 47.05790715022754, "tpot_relative": 1.583263, "ttft_error_pct": 13.83, "ttft_ms": 1001.4446097750008, "ttft_relative": 1.391174, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 43374.22715897801, "interactivity_tok_s": 21.80405546362787, "output_per_gpu": 1337.1264999440202, "total_per_gpu": 12071.687478836755, "tpot_ms": 45.863027713726744, "tpot_relative": 1.543061, "ttft_ms": 1162.183033710653, "ttft_relative": 1.614467, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 79905.30405971689, "interactivity_tok_s": 12.873104330728404, "output_per_gpu": 1638.7397750484695, "total_per_gpu": null, "tpot_error_pct": 3.53, "tpot_ms": 77.68134043728492, "tpot_relative": 2.613588, "ttft_error_pct": 79.52, "ttft_ms": 437.2927923744137, "ttft_relative": 0.607473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 73346.73313807006, "interactivity_tok_s": 12.912409552729601, "output_per_gpu": 1589.2146844614565, "total_per_gpu": 14298.819848160292, "tpot_error_pct": 3.21, "tpot_ms": 77.4448793555039, "tpot_relative": 2.605632, "ttft_error_pct": 11.35, "ttft_ms": 1892.9813300175836, "ttft_relative": 2.629668, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 71356.41707149644, "interactivity_tok_s": 13.327116012900033, "output_per_gpu": 1634.3518410432337, "total_per_gpu": 14704.937458783646, "tpot_ms": 75.03498874265416, "tpot_relative": 2.524551, "ttft_ms": 2135.247648425775, "ttft_relative": 2.966216, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.21, "tpot_shape_error_pct": 2.47, "ttft_mape_pct": 10.91, "ttft_shape_error_pct": 19.48}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.96, "tpot_shape_error_pct": 2.19, "ttft_mape_pct": 19.81, "ttft_shape_error_pct": 4.3}, "framework": "vllm", "id": "f2e5b03940b1ff84", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 12077.642799865893, "interactivity_tok_s": 87.21840009369666, "output_per_gpu": 338.80783426095667, "total_per_gpu": null, "tpot_error_pct": 13.73, "tpot_ms": 11.465470576457763, "tpot_relative": 1.137334, "ttft_error_pct": 14.1, "ttft_ms": 348.4664001496003, "ttft_relative": 1.14098, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10479.210213075014, "interactivity_tok_s": 89.86694365841653, "output_per_gpu": 343.65107310874384, "total_per_gpu": 3089.218198434817, "tpot_error_pct": 10.38, "tpot_ms": 11.127562141213918, "tpot_relative": 1.103814, "ttft_error_pct": 17.44, "ttft_ms": 252.14071204999976, "ttft_relative": 0.825582, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 9569.835478697496, "interactivity_tok_s": 99.19641394572906, "output_per_gpu": 376.4235953482048, "total_per_gpu": 3383.8236282823063, "tpot_ms": 10.081009587172233, "tpot_relative": 1.0, "ttft_ms": 305.40973237511935, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17145.662282903624, "interactivity_tok_s": 61.03399064902136, "output_per_gpu": 477.32189430562124, "total_per_gpu": null, "tpot_error_pct": 17.26, "tpot_ms": 16.38431289460563, "tpot_relative": 1.625265, "ttft_error_pct": 1.2, "ttft_ms": 384.5101917220668, "ttft_relative": 1.258998, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15007.563845049974, "interactivity_tok_s": 62.96105775792071, "output_per_gpu": 486.72943257631636, "total_per_gpu": 4329.683336848319, "tpot_error_pct": 13.67, "tpot_ms": 15.882833542042846, "tpot_relative": 1.57552, "ttft_error_pct": 20.55, "ttft_ms": 301.89210482500084, "ttft_relative": 0.988482, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 13318.28451040783, "interactivity_tok_s": 71.56575581943935, "output_per_gpu": 548.1274387456361, "total_per_gpu": 4875.846988427646, "tpot_ms": 13.973163401264197, "tpot_relative": 1.386088, "ttft_ms": 379.968645205372, "ttft_relative": 1.244127, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23863.4567910954, "interactivity_tok_s": 43.63989242837293, "output_per_gpu": 685.9023042339652, "total_per_gpu": null, "tpot_error_pct": 11.65, "tpot_ms": 22.914813588079323, "tpot_relative": 2.273067, "ttft_error_pct": 17.45, "ttft_ms": 421.6024904902506, "ttft_relative": 1.380449, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20836.66965296249, "interactivity_tok_s": 44.76685594654856, "output_per_gpu": 686.8624503553142, "total_per_gpu": 6198.761694792132, "tpot_error_pct": 8.84, "tpot_ms": 22.337954695634554, "tpot_relative": 2.215845, "ttft_error_pct": 21.44, "ttft_ms": 401.1899314500006, "ttft_relative": 1.313612, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26006186869", "measured": {"e2e_ms": 19284.957876449334, "interactivity_tok_s": 48.72312601132311, "output_per_gpu": 741.8074102511405, "total_per_gpu": 6694.626205289031, "tpot_ms": 20.524134674109437, "tpot_relative": 2.035921, "ttft_ms": 510.70241994493705, "ttft_relative": 1.672188, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}]}, {"aic": {"coverage_pct": 33.33, "points": 4, "status_counts": {"failed": 8, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.28, "tpot_shape_error_pct": 5.09, "ttft_mape_pct": 45.65, "ttft_shape_error_pct": 33.45}, "aisimulate": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.89, "tpot_shape_error_pct": 3.67, "ttft_mape_pct": 13.0, "ttft_shape_error_pct": 7.53}, "gpu": "gb200", "precisions": ["fp8"], "rows": 12, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.72, "tpot_shape_error_pct": 0.58, "ttft_mape_pct": 3.14, "ttft_shape_error_pct": 7.37}, "framework": "dynamo-vllm", "id": "52b5478bc61cdd72", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42900.8276668043, "interactivity_tok_s": 33.82406519339236, "output_per_gpu": 1514.1945917218713, "total_per_gpu": 13624.956480331903, "tpot_error_pct": 8.36, "tpot_ms": 29.56474907088795, "tpot_relative": 0.916408, "ttft_error_pct": 7.62, "ttft_ms": 15676.933762106355, "ttft_relative": 1.076179, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 44272.39174743365, "interactivity_tok_s": 30.996637113579254, "output_per_gpu": 2567.943364180228, "total_per_gpu": 13209.535604069859, "tpot_ms": 32.26156425730171, "tpot_relative": 1.0, "ttft_ms": 14567.221461381063, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 83193.92623733259, "interactivity_tok_s": 33.43826926561735, "output_per_gpu": 1552.7891074088493, "total_per_gpu": 13962.418556089207, "tpot_error_pct": 8.8, "tpot_ms": 29.905854039767615, "tpot_relative": 0.926981, "ttft_error_pct": 0.58, "ttft_ms": 55636.90716794486, "ttft_relative": 3.819322, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 85532.36920596921, "interactivity_tok_s": 30.49556910429498, "output_per_gpu": 2641.9383599883877, "total_per_gpu": 13581.793473222888, "tpot_ms": 32.79164906154057, "tpot_relative": 1.016431, "ttft_ms": 55315.95869293734, "ttft_relative": 3.79729, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 163676.78659279484, "interactivity_tok_s": 33.29443822104902, "output_per_gpu": 1571.1023077442992, "total_per_gpu": 14143.0224522068, "tpot_error_pct": 8.99, "tpot_ms": 30.035046495176836, "tpot_relative": 0.930985, "ttft_error_pct": 1.21, "ttft_ms": 136032.60844269636, "ttft_relative": 9.338267, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 168075.70409236217, "interactivity_tok_s": 30.301546908737024, "output_per_gpu": 2675.994642193556, "total_per_gpu": 13772.1702511646, "tpot_ms": 33.001615495467135, "tpot_relative": 1.022939, "ttft_ms": 137702.80568011696, "ttft_relative": 9.452922, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 57.14, "points": 4, "status_counts": {"failed": 3, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.28, "tpot_shape_error_pct": 5.09, "ttft_mape_pct": 45.65, "ttft_shape_error_pct": 33.45}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.27, "tpot_shape_error_pct": 4.02, "ttft_mape_pct": 17.33, "ttft_shape_error_pct": 7.39}, "framework": "dynamo-vllm", "id": "eace6563fc51f098", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8229.076, "interactivity_tok_s": 133.24450366422386, "output_per_gpu": 20.411666666666665, "total_per_gpu": null, "tpot_error_pct": 7.19, "tpot_ms": 7.505, "tpot_relative": 1.07189, "ttft_error_pct": 89.3, "ttft_ms": 551.461, "ttft_relative": 1.893031, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6697.845637999986, "interactivity_tok_s": 144.26195283963207, "output_per_gpu": 23.149234601694825, "total_per_gpu": 207.22086001190348, "tpot_error_pct": 1.0, "tpot_ms": 6.931834626636754, "tpot_relative": 0.990028, "ttft_error_pct": 12.11, "ttft_ms": 256.0329160000002, "ttft_relative": 0.878898, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 6798.0761498445645, "interactivity_tok_s": 142.8234134007293, "output_per_gpu": 34.20995981069388, "total_per_gpu": 204.35023898958752, "tpot_ms": 7.001653133679367, "tpot_relative": 1.0, "ttft_ms": 291.31113239564, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9656.161, "interactivity_tok_s": 112.35955056179775, "output_per_gpu": 68.92883333333333, "total_per_gpu": null, "tpot_error_pct": 9.45, "tpot_ms": 8.9, "tpot_relative": 1.271128, "ttft_error_pct": 56.35, "ttft_ms": 551.461, "ttft_relative": 1.893031, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7828.3217644749875, "interactivity_tok_s": 122.03674467158052, "output_per_gpu": 76.35553140996866, "total_per_gpu": 686.3906899781329, "tpot_error_pct": 0.77, "tpot_ms": 8.194253318466929, "tpot_relative": 1.170331, "ttft_error_pct": 15.84, "ttft_ms": 296.8357915250006, "ttft_relative": 1.018965, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 7826.759817567654, "interactivity_tok_s": 122.97703835063751, "output_per_gpu": 114.09090755402984, "total_per_gpu": 684.4007844804752, "tpot_ms": 8.131599308390857, "tpot_relative": 1.161383, "ttft_ms": 352.71226632175967, "ttft_relative": 1.210775, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10691.437, "interactivity_tok_s": 100.88781275221952, "output_per_gpu": 123.88350000000001, "total_per_gpu": null, "tpot_error_pct": 15.47, "tpot_ms": 9.912, "tpot_relative": 1.415666, "ttft_error_pct": 29.29, "ttft_ms": 551.461, "ttft_relative": 1.893031, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8866.101649225018, "interactivity_tok_s": 108.63927191642128, "output_per_gpu": 136.580231099535, "total_per_gpu": 1214.9443020210808, "tpot_error_pct": 7.23, "tpot_ms": 9.204774501520253, "tpot_relative": 1.314657, "ttft_error_pct": 17.35, "ttft_ms": 352.5112791499997, "ttft_relative": 1.210085, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 8365.747816485236, "interactivity_tok_s": 116.49549781239554, "output_per_gpu": 216.8113002927001, "total_per_gpu": 1287.0067562807742, "tpot_ms": 8.584022720005892, "tpot_relative": 1.225999, "ttft_ms": 426.53500328597147, "ttft_relative": 1.464191, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11789.116, "interactivity_tok_s": 91.03322712790168, "output_per_gpu": 223.587, "total_per_gpu": null, "tpot_error_pct": 13.03, "tpot_ms": 10.985, "tpot_relative": 1.568915, "ttft_error_pct": 7.68, "ttft_ms": 551.461, "ttft_relative": 1.893031, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9784.07072656247, "interactivity_tok_s": 98.01742404610683, "output_per_gpu": 240.38540681661547, "total_per_gpu": 2169.4181287549413, "tpot_error_pct": 4.98, "tpot_ms": 10.20226770629685, "tpot_relative": 1.457123, "ttft_error_pct": 23.46, "ttft_ms": 457.2121893249976, "ttft_relative": 1.569498, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 9475.578851706814, "interactivity_tok_s": 102.89484896824608, "output_per_gpu": 372.7602516726255, "total_per_gpu": 2244.1867633120914, "tpot_ms": 9.71865948613818, "tpot_relative": 1.388052, "ttft_ms": 597.323877501185, "ttft_relative": 2.050467, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11720.367080900009, "interactivity_tok_s": 83.59777330092655, "output_per_gpu": 406.47536252129, "total_per_gpu": 3634.832116260363, "tpot_error_pct": 4.83, "tpot_ms": 11.962041098873582, "tpot_relative": 1.70846, "ttft_error_pct": 10.89, "ttft_ms": 663.5819512468743, "ttft_relative": 2.277915, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 12363.110922676424, "interactivity_tok_s": 79.5600013363425, "output_per_gpu": 580.1485766796101, "total_per_gpu": 3461.491205688119, "tpot_ms": 12.569130005069601, "tpot_relative": 1.795166, "ttft_ms": 744.6920655900612, "ttft_relative": 2.556346, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15228.876629459479, "interactivity_tok_s": 64.32818670476583, "output_per_gpu": 624.7834457122302, "total_per_gpu": 5631.659262679718, "tpot_error_pct": 1.8, "tpot_ms": 15.545285064376202, "tpot_relative": 2.220231, "ttft_error_pct": 24.8, "ttft_ms": 917.9564732203132, "ttft_relative": 3.15112, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 15276.86736950891, "interactivity_tok_s": 65.48755974771309, "output_per_gpu": 928.1486035197579, "total_per_gpu": 5581.9111899623285, "tpot_ms": 15.27007578007854, "tpot_relative": 2.180924, "ttft_ms": 1220.7422784555092, "ttft_relative": 4.19051, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19264.78966193126, "interactivity_tok_s": 52.13329064140104, "output_per_gpu": 983.7987266562637, "total_per_gpu": 8881.81542342416, "tpot_error_pct": 2.25, "tpot_ms": 19.18160138554273, "tpot_relative": 2.739582, "ttft_error_pct": 16.84, "ttft_ms": 1637.2342455539085, "ttft_relative": 5.620225, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 20001.922148179074, "interactivity_tok_s": 50.961567208781695, "output_per_gpu": 1423.21228217196, "total_per_gpu": 8569.80899848648, "tpot_ms": 19.622630440369974, "tpot_relative": 2.802571, "ttft_ms": 1968.6903691757834, "ttft_relative": 6.758033, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.84, "tpot_shape_error_pct": 7.8, "ttft_mape_pct": 12.62, "ttft_shape_error_pct": 8.73}, "framework": "dynamo-vllm", "id": "770a30c1112defc0", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 32148.002683609793, "interactivity_tok_s": 46.755227424147435, "output_per_gpu": 1178.4418278387695, "total_per_gpu": 10602.927069927891, "tpot_error_pct": 13.46, "tpot_ms": 21.387982800903565, "tpot_relative": 0.86537, "ttft_error_pct": 17.76, "ttft_ms": 12449.17826373426, "ttft_relative": 1.177573, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 33334.54545549648, "interactivity_tok_s": 40.460584468640064, "output_per_gpu": 1713.135459928725, "total_per_gpu": 10280.112457347395, "tpot_ms": 24.71541163166028, "tpot_relative": 1.0, "ttft_ms": 10571.895265525018, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 62337.96267787204, "interactivity_tok_s": 46.363983776052194, "output_per_gpu": 1207.3360439782539, "total_per_gpu": 10866.112107659617, "tpot_error_pct": 20.21, "tpot_ms": 21.56846583395876, "tpot_relative": 0.872673, "ttft_error_pct": 7.48, "ttft_ms": 42471.00131315576, "ttft_relative": 4.01735, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906115049", "measured": {"e2e_ms": 64411.51118657612, "interactivity_tok_s": 36.9925471985895, "output_per_gpu": 1760.2427959966747, "total_per_gpu": 10567.572638984739, "tpot_ms": 27.03247209854555, "tpot_relative": 1.09375, "ttft_ms": 39514.278617566066, "ttft_relative": 3.737672, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}]}, {"aic": {"coverage_pct": 5.56, "points": 1, "status_counts": {"failed": 17, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.77, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.19, "tpot_shape_error_pct": 9.87, "ttft_mape_pct": 32.78, "ttft_shape_error_pct": 15.34}, "gpu": "gb300", "precisions": ["fp8"], "rows": 18, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.59, "tpot_shape_error_pct": 6.18, "ttft_mape_pct": 39.63, "ttft_shape_error_pct": 19.63}, "framework": "dynamo-vllm", "id": "6b883d515111a39d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15645.06271657178, "interactivity_tok_s": 61.11660600652521, "output_per_gpu": 609.8602016027176, "total_per_gpu": 5497.14445359613, "tpot_error_pct": 5.74, "tpot_ms": 16.362165135499072, "tpot_relative": 1.057356, "ttft_error_pct": 30.99, "ttft_ms": 581.9597001765617, "ttft_relative": 0.690051, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 15087.574103484712, "interactivity_tok_s": 64.62198955718043, "output_per_gpu": 949.6132691546684, "total_per_gpu": 5711.000278543558, "tpot_ms": 15.474608671946804, "tpot_relative": 1.0, "ttft_ms": 843.358092917515, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19905.271786316374, "interactivity_tok_s": 48.41578423407815, "output_per_gpu": 953.6525693237445, "total_per_gpu": 8609.653447708884, "tpot_error_pct": 5.63, "tpot_ms": 20.654421193824955, "tpot_relative": 1.33473, "ttft_error_pct": 27.95, "ttft_ms": 924.3105863406245, "ttft_relative": 1.095988, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 19252.41452120722, "interactivity_tok_s": 51.14298135690039, "output_per_gpu": 1478.244636257705, "total_per_gpu": 8901.183853214532, "tpot_ms": 19.553025135971595, "tpot_relative": 1.263555, "ttft_ms": 1282.9355218114415, "ttft_relative": 1.521223, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26860.64887760068, "interactivity_tok_s": 37.28002333268647, "output_per_gpu": 1418.9313105777094, "total_per_gpu": 12766.710114901862, "tpot_error_pct": 11.28, "tpot_ms": 26.82401754623414, "tpot_relative": 1.733421, "ttft_error_pct": 67.88, "ttft_ms": 2155.310047624991, "ttft_relative": 2.555629, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 28909.031003404587, "interactivity_tok_s": 41.48688221214249, "output_per_gpu": 1973.2566695594007, "total_per_gpu": 11841.03706027158, "tpot_ms": 24.104004607685784, "tpot_relative": 1.557649, "ttft_ms": 6709.966675159899, "ttft_relative": 7.956249, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 49816.254215031324, "interactivity_tok_s": 33.84668516541337, "output_per_gpu": 1521.8183763626469, "total_per_gpu": 13696.475945971926, "tpot_error_pct": 19.69, "tpot_ms": 29.544990746150283, "tpot_relative": 1.909256, "ttft_error_pct": 31.71, "ttft_ms": 22602.495628578323, "ttft_relative": 26.800591, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 55831.03049690247, "interactivity_tok_s": 40.511417552062476, "output_per_gpu": 2027.4976772868708, "total_per_gpu": 12172.03048853854, "tpot_ms": 24.684399125625983, "tpot_relative": 1.595155, "ttft_ms": 33095.8001599104, "ttft_relative": 39.24288, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.81, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.75, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5905a59d53dd2d44", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17327.57350493748, "interactivity_tok_s": 60.665554760931144, "output_per_gpu": 534.3090062871264, "total_per_gpu": 4808.781056584137, "tpot_error_pct": 4.81, "tpot_ms": 16.48381860086449, "tpot_relative": 1.048126, "ttft_error_pct": 33.75, "ttft_ms": 464.6270762531252, "ttft_relative": 0.662475, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 16790.019072879386, "interactivity_tok_s": 63.58512255665044, "output_per_gpu": 963.5458549811902, "total_per_gpu": 4955.378682760407, "tpot_ms": 15.726949320716672, "tpot_relative": 1.0, "ttft_ms": 701.3499177862286, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 7}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.49, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.77, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "96c811dad527fda9", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17925.598537318823, "interactivity_tok_s": 58.36755104963698, "output_per_gpu": 453.4453041742218, "total_per_gpu": 4081.007737567996, "tpot_error_pct": 1.49, "tpot_ms": 17.13280721936713, "tpot_relative": 0.985093, "ttft_error_pct": 41.77, "ttft_ms": 398.7367519062517, "ttft_relative": 0.58228, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 18476.870778925513, "interactivity_tok_s": 57.4974744594009, "output_per_gpu": 878.7582138403111, "total_per_gpu": 3954.4119622814, "tpot_ms": 17.392068249990743, "tpot_relative": 1.0, "ttft_ms": 684.7849591849808, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.3, "tpot_shape_error_pct": 10.4, "ttft_mape_pct": 29.09, "ttft_shape_error_pct": 13.02}, "framework": "dynamo-vllm", "id": "75d61c5425ab3b43", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 25667.536012142125, "interactivity_tok_s": 37.79118443523681, "output_per_gpu": 1365.6013305800645, "total_per_gpu": 12290.511184903706, "tpot_error_pct": 4.06, "tpot_ms": 26.46119762966709, "tpot_relative": 0.959358, "ttft_error_pct": 28.45, "ttft_ms": 1294.946189166802, "ttft_relative": 0.715456, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 27212.67337169876, "interactivity_tok_s": 36.25525648814381, "output_per_gpu": 2091.9501634165686, "total_per_gpu": 11592.894777043024, "tpot_ms": 27.582207295293028, "tpot_relative": 1.0, "ttft_ms": 1809.9594313861644, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 40590.157461425406, "interactivity_tok_s": 30.801951481380115, "output_per_gpu": 1729.0706119336553, "total_per_gpu": 15558.444051914932, "tpot_error_pct": 5.72, "tpot_ms": 32.465475461985044, "tpot_relative": 1.177044, "ttft_error_pct": 38.73, "ttft_ms": 10695.160788136563, "ttft_relative": 5.909061, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 45729.045572903786, "interactivity_tok_s": 32.56457195726185, "output_per_gpu": 2482.5669044937044, "total_per_gpu": 13752.693159513428, "tpot_ms": 30.7082187756809, "tpot_relative": 1.113334, "ttft_ms": 17454.46195657621, "ttft_relative": 9.643565, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 78400.31957233963, "interactivity_tok_s": 30.3114649679971, "output_per_gpu": 1778.5769792353278, "total_per_gpu": 15992.665133868588, "tpot_error_pct": 6.1, "tpot_ms": 32.9908172058263, "tpot_relative": 1.196091, "ttft_error_pct": 20.1, "ttft_ms": 48000.73128372177, "ttft_relative": 26.520335, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 88725.74662807878, "interactivity_tok_s": 32.16109675843561, "output_per_gpu": 2545.852568216587, "total_per_gpu": 14094.58764933872, "tpot_ms": 31.09346697692166, "tpot_relative": 1.127302, "ttft_ms": 60074.48980593508, "ttft_relative": 33.19107, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 20.0, "points": 1, "status_counts": {"failed": 4, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.77, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.57, "tpot_shape_error_pct": 5.51, "ttft_mape_pct": 29.65, "ttft_shape_error_pct": 16.43}, "framework": "dynamo-vllm", "id": "01d9bd764b724ab6", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12206.324, "interactivity_tok_s": 86.84324793747285, "output_per_gpu": 255.869, "total_per_gpu": null, "tpot_error_pct": 16.23, "tpot_ms": 11.515, "tpot_relative": 1.162341, "ttft_error_pct": 36.77, "ttft_ms": 426.479, "ttft_relative": 0.632256, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10179.898861049971, "interactivity_tok_s": 93.69282147982176, "output_per_gpu": 276.8989751096291, "total_per_gpu": 2498.9439433599237, "tpot_error_pct": 7.74, "tpot_ms": 10.673176281870921, "tpot_relative": 1.077366, "ttft_error_pct": 37.33, "ttft_ms": 422.73107434374714, "ttft_relative": 0.6267, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 9724.816207726326, "interactivity_tok_s": 100.94147535045371, "output_per_gpu": 362.62174090304325, "total_per_gpu": 2619.7779641113784, "tpot_ms": 9.906730573613567, "tpot_relative": 1.0, "ttft_ms": 674.5354422131641, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16055.089608607776, "interactivity_tok_s": 61.502922020780765, "output_per_gpu": 710.2384663785178, "total_per_gpu": 6401.931845893209, "tpot_error_pct": 5.43, "tpot_ms": 16.259390076818097, "tpot_relative": 1.641247, "ttft_error_pct": 36.02, "ttft_ms": 1087.4389251624957, "ttft_relative": 1.61213, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 15894.530411188225, "interactivity_tok_s": 64.8449881253463, "output_per_gpu": 895.1289038301948, "total_per_gpu": 6459.995771108797, "tpot_ms": 15.421392291212783, "tpot_relative": 1.556658, "ttft_ms": 1699.77762508679, "ttft_relative": 2.519923, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 25265.358624994587, "interactivity_tok_s": 54.04132381454457, "output_per_gpu": 897.2546157797939, "total_per_gpu": 8100.4881072141525, "tpot_error_pct": 14.47, "tpot_ms": 18.504357950810636, "tpot_relative": 1.867857, "ttft_error_pct": 38.71, "ttft_ms": 8259.672318755493, "ttft_relative": 12.244979, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 28333.94972693418, "interactivity_tok_s": 61.859308129796155, "output_per_gpu": 998.7211947109913, "total_per_gpu": 7216.505918584502, "tpot_ms": 16.165715883885287, "tpot_relative": 1.631791, "ttft_ms": 13475.707648850675, "ttft_relative": 19.977761, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 48813.98885953616, "interactivity_tok_s": 53.693497852009195, "output_per_gpu": 926.6372952038786, "total_per_gpu": 8337.337855141206, "tpot_error_pct": 15.26, "tpot_ms": 18.62422900359769, "tpot_relative": 1.879957, "ttft_error_pct": 20.65, "ttft_ms": 31660.673658770225, "ttft_relative": 46.937005, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 54784.15804299412, "interactivity_tok_s": 61.887103859974346, "output_per_gpu": 1029.4412267367507, "total_per_gpu": 7412.914035080743, "tpot_ms": 16.15845527789761, "tpot_relative": 1.631058, "ttft_ms": 39901.70800522426, "ttft_relative": 59.154354, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 96022.33827349448, "interactivity_tok_s": 53.523113647646454, "output_per_gpu": 938.1687244214861, "total_per_gpu": 8443.58667689002, "tpot_error_pct": 14.94, "tpot_ms": 18.683516930334125, "tpot_relative": 1.885942, "ttft_error_pct": 15.51, "ttft_ms": 78812.986601728, "ttft_relative": 116.840394, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 108253.80421941099, "interactivity_tok_s": 61.51901941470601, "output_per_gpu": 1039.1730620401088, "total_per_gpu": 7486.378703592745, "tpot_ms": 16.255135558304296, "tpot_relative": 1.640817, "ttft_ms": 93282.9951991462, "ttft_relative": 138.292207, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.78, "tpot_shape_error_pct": 19.01, "ttft_mape_pct": 30.12, "ttft_shape_error_pct": 11.14}, "framework": "dynamo-vllm", "id": "46808b3dc2e0b98e", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 28361.152388255134, "interactivity_tok_s": 37.611319867195505, "output_per_gpu": 1285.2004600699881, "total_per_gpu": 11566.804140629893, "tpot_error_pct": 7.86, "tpot_ms": 26.58774016787955, "tpot_relative": 0.921431, "ttft_error_pct": 26.28, "ttft_ms": 1161.8941965144654, "ttft_relative": 0.73717, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 31094.636491406163, "interactivity_tok_s": 34.656253632949465, "output_per_gpu": 2049.6181629135403, "total_per_gpu": 10540.893409269638, "tpot_ms": 28.85482114111864, "tpot_relative": 1.0, "ttft_ms": 1576.1544640417924, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41496.83553573636, "interactivity_tok_s": 26.231119204190914, "output_per_gpu": 1754.675272734494, "total_per_gpu": 15792.077454610448, "tpot_error_pct": 3.16, "tpot_ms": 38.12265851928389, "tpot_relative": 1.321189, "ttft_error_pct": 34.72, "ttft_ms": 2497.3558705109317, "ttft_relative": 1.584461, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 44096.03559880446, "interactivity_tok_s": 25.403088376730842, "output_per_gpu": 2889.3001561519513, "total_per_gpu": 14859.257945924322, "tpot_ms": 39.36529232862872, "tpot_relative": 1.364254, "ttft_ms": 3825.3415466172783, "ttft_relative": 2.427009, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 74564.43102654039, "interactivity_tok_s": 20.989997890950814, "output_per_gpu": 1948.9350102760432, "total_per_gpu": 17540.41509248439, "tpot_error_pct": 15.5, "tpot_ms": 47.64173894610627, "tpot_relative": 1.651084, "ttft_error_pct": 37.85, "ttft_ms": 25826.932084674143, "ttft_relative": 16.386041, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 83752.04344137739, "interactivity_tok_s": 24.243363100142055, "output_per_gpu": 3017.6657508523595, "total_per_gpu": 15519.42386152642, "tpot_ms": 41.2484025367809, "tpot_relative": 1.429515, "ttft_ms": 41554.92764625054, "ttft_relative": 26.364756, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 144614.7125856517, "interactivity_tok_s": 20.491831891575252, "output_per_gpu": 1992.221963645056, "total_per_gpu": 17929.997672805504, "tpot_error_pct": 16.62, "tpot_ms": 48.79993186022217, "tpot_relative": 1.691223, "ttft_error_pct": 21.65, "ttft_ms": 94692.38229264595, "ttft_relative": 60.078111, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26906377482", "measured": {"e2e_ms": 163659.18737895548, "interactivity_tok_s": 23.898123370463487, "output_per_gpu": 3068.9377405245405, "total_per_gpu": 15783.108379840492, "tpot_ms": 41.844289800425685, "tpot_relative": 1.450166, "ttft_ms": 120852.47891312001, "ttft_relative": 76.67553, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 7}]}, {"aic": {"coverage_pct": 90.91, "points": 10, "status_counts": {"failed": 1, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.58, "tpot_shape_error_pct": 3.72, "ttft_mape_pct": 33.51, "ttft_shape_error_pct": 42.18}, "aisimulate": {"coverage_pct": 100.0, "points": 11, "status_counts": {"failed": 0, "success": 11, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.1, "tpot_shape_error_pct": 3.91, "ttft_mape_pct": 31.11, "ttft_shape_error_pct": 10.65}, "gpu": "h100", "precisions": ["fp8"], "rows": 11, "topologies": [{"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.32, "tpot_shape_error_pct": 2.51, "ttft_mape_pct": 32.0, "ttft_shape_error_pct": 42.76}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.63, "tpot_shape_error_pct": 3.63, "ttft_mape_pct": 31.73, "ttft_shape_error_pct": 13.1}, "framework": "vllm", "id": "f8142f43896742a6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12228.353649474435, "interactivity_tok_s": 85.81749494324058, "output_per_gpu": 83.6580319251699, "total_per_gpu": null, "tpot_error_pct": 6.14, "tpot_ms": 11.652635638705101, "tpot_relative": 1.061377, "ttft_error_pct": 0.38, "ttft_ms": 307.7073910791172, "ttft_relative": 1.003808, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10630.981339850066, "interactivity_tok_s": 88.23047954662628, "output_per_gpu": 84.67067666323213, "total_per_gpu": 761.1388867657565, "tpot_error_pct": 3.23, "tpot_ms": 11.333951771978525, "tpot_relative": 1.032349, "ttft_error_pct": 30.37, "ttft_ms": 213.4380390999999, "ttft_relative": 0.696281, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 10396.901880548103, "interactivity_tok_s": 91.08468696887647, "output_per_gpu": 86.52921770247423, "total_per_gpu": 777.8460623000166, "tpot_ms": 10.97879383766998, "tpot_relative": 1.0, "ttft_ms": 306.54010919388384, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15929.389065391842, "interactivity_tok_s": 65.62097336760377, "output_per_gpu": 128.44183738628968, "total_per_gpu": null, "tpot_error_pct": 6.5, "tpot_ms": 15.23903027768386, "tpot_relative": 1.388042, "ttft_error_pct": 11.23, "ttft_ms": 339.86109132125415, "ttft_relative": 1.1087, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13935.507988975078, "interactivity_tok_s": 67.71784271921247, "output_per_gpu": 130.92845337523704, "total_per_gpu": 1164.669126125227, "tpot_error_pct": 3.2, "tpot_ms": 14.767156776485535, "tpot_relative": 1.345062, "ttft_error_pct": 31.3, "ttft_ms": 263.0171358374997, "ttft_relative": 0.858019, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 13632.218675719923, "interactivity_tok_s": 69.88690491341963, "output_per_gpu": 133.7609422147352, "total_per_gpu": 1189.8654239231028, "tpot_ms": 14.308832266057054, "tpot_relative": 1.303316, "ttft_ms": 382.8716922114836, "ttft_relative": 1.24901, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20975.849319884608, "interactivity_tok_s": 49.653753194395854, "output_per_gpu": 195.0814928919651, "total_per_gpu": null, "tpot_error_pct": 3.43, "tpot_ms": 20.139464505028887, "tpot_relative": 1.834397, "ttft_error_pct": 25.3, "ttft_ms": 373.1771312400546, "ttft_relative": 1.217384, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18283.154229606145, "interactivity_tok_s": 50.9877717667071, "output_per_gpu": 195.2128515669957, "total_per_gpu": 1761.7471241857254, "tpot_error_pct": 0.73, "tpot_ms": 19.612545623202905, "tpot_relative": 1.786403, "ttft_error_pct": 31.52, "ttft_ms": 342.10423423749916, "ttft_relative": 1.116018, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 18310.280763667834, "interactivity_tok_s": 51.35803774281175, "output_per_gpu": 194.70221913834246, "total_per_gpu": 1757.1387943269349, "tpot_ms": 19.471148898011847, "tpot_relative": 1.773524, "ttft_ms": 499.5363056601491, "ttft_relative": 1.629595, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30123.523199576444, "interactivity_tok_s": 34.42784525618593, "output_per_gpu": 271.68136827086255, "total_per_gpu": null, "tpot_error_pct": 1.2, "tpot_ms": 29.04625580133633, "tpot_relative": 2.645669, "ttft_error_pct": 91.1, "ttft_ms": 409.20351480937995, "ttft_relative": 1.33491, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28088.002149643766, "interactivity_tok_s": 35.89498225405828, "output_per_gpu": 258.7967585342322, "total_per_gpu": 2314.2430175089276, "tpot_error_pct": 2.94, "tpot_ms": 27.85904706463367, "tpot_relative": 2.537533, "ttft_error_pct": 49.93, "ttft_ms": 2301.893575418755, "ttft_relative": 7.509274, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 31130.0042055198, "interactivity_tok_s": 34.841198928164076, "output_per_gpu": 231.2431474516852, "total_per_gpu": 2067.8498539465395, "tpot_ms": 28.701652950055184, "tpot_relative": 2.614281, "ttft_ms": 4597.092551674723, "ttft_relative": 14.996708, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 55122.19351973594, "interactivity_tok_s": 35.65405502591625, "output_per_gpu": 258.7300545235499, "total_per_gpu": 2332.13526717271, "tpot_error_pct": 3.06, "tpot_ms": 28.04730063026826, "tpot_relative": 2.55468, "ttft_error_pct": 15.51, "ttft_ms": 29271.677125567203, "ttft_relative": 95.490529, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956498305", "measured": {"e2e_ms": 61310.12193657698, "interactivity_tok_s": 34.562121042641024, "output_per_gpu": 231.9310477561306, "total_per_gpu": 2090.5749701960403, "tpot_ms": 28.933409461943892, "tpot_relative": 2.635391, "ttft_ms": 34645.53336362478, "ttft_relative": 113.021208, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.75, "tpot_shape_error_pct": 4.44, "ttft_mape_pct": 34.51, "ttft_shape_error_pct": 41.83}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.49, "tpot_shape_error_pct": 4.13, "ttft_mape_pct": 30.6, "ttft_shape_error_pct": 8.69}, "framework": "vllm", "id": "0f667d12cb6ac5ba", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10013.132267889545, "interactivity_tok_s": 104.67663157446995, "output_per_gpu": 51.08291654553448, "total_per_gpu": null, "tpot_error_pct": 6.77, "tpot_ms": 9.553230601316889, "tpot_relative": 1.0677, "ttft_error_pct": 1.09, "ttft_ms": 240.17736274236773, "ttft_relative": 1.010895, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8764.154433724996, "interactivity_tok_s": 106.77204014888724, "output_per_gpu": 51.30629567105341, "total_per_gpu": 461.2129997078217, "tpot_error_pct": 4.67, "tpot_ms": 9.365747798820362, "tpot_relative": 1.046746, "ttft_error_pct": 34.45, "ttft_ms": 155.735788675, "ttft_relative": 0.655485, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 8460.352655223687, "interactivity_tok_s": 111.76319533886964, "output_per_gpu": 53.1115449119609, "total_per_gpu": 477.44111375755534, "tpot_ms": 8.947489349852315, "tpot_relative": 1.0, "ttft_ms": 237.5888031310751, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12401.598532510681, "interactivity_tok_s": 84.28720097135609, "output_per_gpu": 82.4893659731215, "total_per_gpu": null, "tpot_error_pct": 8.09, "tpot_ms": 11.864197511314169, "tpot_relative": 1.325981, "ttft_error_pct": 12.11, "ttft_ms": 264.5244784362873, "ttft_relative": 1.113371, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10924.516331325065, "interactivity_tok_s": 86.2543126803506, "output_per_gpu": 83.42557005629573, "total_per_gpu": 742.1090165595866, "tpot_error_pct": 5.63, "tpot_ms": 11.593623193148552, "tpot_relative": 1.29574, "ttft_error_pct": 36.3, "ttft_ms": 191.7059799124998, "ttft_relative": 0.806881, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 10461.980648100143, "interactivity_tok_s": 91.11007883866851, "output_per_gpu": 87.08900418895672, "total_per_gpu": 774.6969569187032, "tpot_ms": 10.97573410918381, "tpot_relative": 1.226683, "ttft_ms": 300.9722681250423, "ttft_relative": 1.266778, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15764.217931277391, "interactivity_tok_s": 66.1113165044867, "output_per_gpu": 129.78759929096023, "total_per_gpu": null, "tpot_error_pct": 5.76, "tpot_ms": 15.126003426843486, "tpot_relative": 1.69053, "ttft_error_pct": 24.28, "ttft_ms": 290.3164256165061, "ttft_relative": 1.221928, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13768.986066143745, "interactivity_tok_s": 67.65483579666697, "output_per_gpu": 129.26667281335483, "total_per_gpu": 1166.5993670699893, "tpot_error_pct": 3.34, "tpot_ms": 14.780909423909431, "tpot_relative": 1.651962, "ttft_error_pct": 35.28, "ttft_ms": 248.14079498125028, "ttft_relative": 1.044413, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 13467.890761172748, "interactivity_tok_s": 69.91636243713245, "output_per_gpu": 132.26126796737123, "total_per_gpu": 1193.624838796569, "tpot_ms": 14.302803594783443, "tpot_relative": 1.598527, "ttft_ms": 383.42195821751375, "ttft_relative": 1.613805, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21305.88440533051, "interactivity_tok_s": 48.74046864383908, "output_per_gpu": 192.05961705941783, "total_per_gpu": null, "tpot_error_pct": 3.04, "tpot_ms": 20.51683186116436, "tpot_relative": 2.293027, "ttft_error_pct": 40.28, "ttft_ms": 317.1654113593719, "ttft_relative": 1.334934, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18967.613159287477, "interactivity_tok_s": 49.734181221095184, "output_per_gpu": 190.66396878929797, "total_per_gpu": 1704.9779176535078, "tpot_error_pct": 0.98, "tpot_ms": 20.106895809834732, "tpot_relative": 2.247211, "ttft_error_pct": 31.75, "ttft_ms": 362.4789315562499, "ttft_relative": 1.525657, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 18956.40710068692, "interactivity_tok_s": 50.22374199659868, "output_per_gpu": 190.7163952980414, "total_per_gpu": 1705.4467321876582, "tpot_ms": 19.910901901091385, "tpot_relative": 2.225306, "ttft_ms": 531.1044909089105, "ttft_relative": 2.235394, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30735.530078509946, "interactivity_tok_s": 33.66429203801055, "output_per_gpu": 266.2716399910796, "total_per_gpu": null, "tpot_error_pct": 0.35, "tpot_ms": 29.70506549999311, "tpot_relative": 3.319933, "ttft_error_pct": 58.96, "ttft_ms": 347.2480720169955, "ttft_relative": 1.461551, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27628.92725435801, "interactivity_tok_s": 34.097051874707184, "output_per_gpu": 262.22818711840324, "total_per_gpu": 2363.6666577130436, "tpot_error_pct": 0.92, "tpot_ms": 29.328048761359014, "tpot_relative": 3.277796, "ttft_error_pct": 29.81, "ttft_ms": 593.8679943906241, "ttft_relative": 2.499562, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 28132.92462784666, "interactivity_tok_s": 33.78345267950151, "output_per_gpu": 257.58740140423953, "total_per_gpu": 2321.8356456517554, "tpot_ms": 29.60029010317117, "tpot_relative": 3.308223, "ttft_ms": 846.1374242666352, "ttft_relative": 3.561352, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 46021.75110809736, "interactivity_tok_s": 22.415670411423115, "output_per_gpu": 355.6579140492572, "total_per_gpu": null, "tpot_error_pct": 4.47, "tpot_ms": 44.61164808572471, "tpot_relative": 4.98594, "ttft_error_pct": 70.36, "ttft_ms": 384.03511640098185, "ttft_relative": 1.616386, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41764.101261171934, "interactivity_tok_s": 22.626941681015932, "output_per_gpu": 347.08867409606506, "total_per_gpu": 3133.544957269888, "tpot_error_pct": 5.36, "tpot_ms": 44.19510219708582, "tpot_relative": 4.939386, "ttft_error_pct": 16.0, "ttft_ms": 1088.2695906460965, "ttft_relative": 4.580475, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 44264.211204522144, "interactivity_tok_s": 21.413357830448597, "output_per_gpu": 327.54740618799354, "total_per_gpu": 2957.1247912374765, "tpot_ms": 46.699822041831105, "tpot_relative": 5.219321, "ttft_ms": 1295.5579528799717, "ttft_relative": 5.452942, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.16, "tpot_shape_error_pct": 5.23, "ttft_mape_pct": 32.92, "ttft_shape_error_pct": 36.03}, "aisimulate": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.64, "tpot_shape_error_pct": 4.9, "ttft_mape_pct": 29.47, "ttft_shape_error_pct": 10.97}, "gpu": "h200", "precisions": ["fp8"], "rows": 15, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.6, "tpot_shape_error_pct": 4.04, "ttft_mape_pct": 33.78, "ttft_shape_error_pct": 35.46}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.26, "tpot_shape_error_pct": 3.55, "ttft_mape_pct": 29.84, "ttft_shape_error_pct": 5.94}, "framework": "vllm", "id": "fc47a7ac7a372845", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7943.902010812486, "interactivity_tok_s": 132.5341748112158, "output_per_gpu": 32.19450588034663, "total_per_gpu": null, "tpot_error_pct": 0.9, "tpot_ms": 7.545223723801193, "tpot_relative": 0.990968, "ttft_error_pct": 4.88, "ttft_ms": 225.13814136386537, "ttft_relative": 0.951166, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7151.638902500008, "interactivity_tok_s": 132.91590505846386, "output_per_gpu": 32.52051776812983, "total_per_gpu": 291.10809821119847, "tpot_error_pct": 1.19, "tpot_ms": 7.523554081508484, "tpot_relative": 0.988122, "ttft_error_pct": 32.43, "ttft_ms": 159.9256734999988, "ttft_relative": 0.675655, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 7312.617384176701, "interactivity_tok_s": 131.33716511570555, "output_per_gpu": 31.801551465478333, "total_per_gpu": 284.67225624413345, "tpot_ms": 7.613991052106378, "tpot_relative": 1.0, "ttft_ms": 236.69709404930472, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9099.166562576898, "interactivity_tok_s": 115.68872837899056, "output_per_gpu": 56.21393964846187, "total_per_gpu": null, "tpot_error_pct": 2.31, "tpot_ms": 8.643884447619213, "tpot_relative": 1.135263, "ttft_error_pct": 0.45, "ttft_ms": 256.4727726624436, "ttft_relative": 1.083548, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8070.354544500001, "interactivity_tok_s": 118.41768938868294, "output_per_gpu": 57.7785832052351, "total_per_gpu": 514.8919677036853, "tpot_error_pct": 0.05, "tpot_ms": 8.44468427953948, "tpot_relative": 1.109101, "ttft_error_pct": 29.73, "ttft_ms": 179.42539780000087, "ttft_relative": 0.758038, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 8150.275449361652, "interactivity_tok_s": 118.36256265750514, "output_per_gpu": 57.21492448498404, "total_per_gpu": 509.8689413246461, "tpot_ms": 8.448617346124957, "tpot_relative": 1.109617, "ttft_ms": 255.33636170439422, "ttft_relative": 1.078747, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10550.879304510936, "interactivity_tok_s": 99.65131842843283, "output_per_gpu": 96.95874348241529, "total_per_gpu": null, "tpot_error_pct": 6.42, "tpot_ms": 10.034990161401385, "tpot_relative": 1.317967, "ttft_error_pct": 7.38, "ttft_ms": 285.08436939731763, "ttft_relative": 1.204427, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9187.856967324962, "interactivity_tok_s": 102.35490388648867, "output_per_gpu": 97.85320991378892, "total_per_gpu": 879.6419988052339, "tpot_error_pct": 3.61, "tpot_ms": 9.769927595349975, "tpot_relative": 1.283155, "ttft_error_pct": 32.32, "ttft_ms": 208.29504412500074, "ttft_relative": 0.880007, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 8978.728340612724, "interactivity_tok_s": 106.0484347022109, "output_per_gpu": 99.82284995032126, "total_per_gpu": 897.3478880672066, "tpot_ms": 9.429653561677247, "tpot_relative": 1.238464, "ttft_ms": 307.78611465357244, "ttft_relative": 1.300338, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13011.628446097488, "interactivity_tok_s": 80.5671856658915, "output_per_gpu": 157.24396131320876, "total_per_gpu": null, "tpot_error_pct": 6.75, "tpot_ms": 12.412001135883722, "tpot_relative": 1.630157, "ttft_error_pct": 10.21, "ttft_ms": 314.1512840884405, "ttft_relative": 1.327229, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11415.880857187556, "interactivity_tok_s": 82.8459159610962, "output_per_gpu": 159.6860248535426, "total_per_gpu": 1420.48102017649, "tpot_error_pct": 3.81, "tpot_ms": 12.070601047728077, "tpot_relative": 1.585319, "ttft_error_pct": 31.18, "ttft_ms": 240.78218464999958, "ttft_relative": 1.017259, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 11115.335852617864, "interactivity_tok_s": 86.00178633706061, "output_per_gpu": 163.6250679351239, "total_per_gpu": 1455.5206295610656, "tpot_ms": 11.627665454305472, "tpot_relative": 1.527145, "ttft_ms": 349.85453891567886, "ttft_relative": 1.478069, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17803.365658509043, "interactivity_tok_s": 58.59405077287507, "output_per_gpu": 229.84418106608095, "total_per_gpu": null, "tpot_error_pct": 7.99, "tpot_ms": 17.06657906066685, "tpot_relative": 2.241476, "ttft_error_pct": 30.82, "ttft_ms": 344.2552794468553, "ttft_relative": 1.454413, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15511.904774787454, "interactivity_tok_s": 60.19928816664067, "output_per_gpu": 229.91591579461232, "total_per_gpu": 2074.933592764378, "tpot_error_pct": 5.11, "tpot_ms": 16.611492103226368, "tpot_relative": 2.181706, "ttft_error_pct": 36.82, "ttft_ms": 314.3708198312499, "ttft_relative": 1.328157, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 14956.987510900944, "interactivity_tok_s": 63.2762214208027, "output_per_gpu": 238.1158363464122, "total_per_gpu": 2148.935823328215, "tpot_ms": 15.803724962490252, "tpot_relative": 2.075616, "ttft_ms": 497.60214287089184, "ttft_relative": 2.102274, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23833.54551993246, "interactivity_tok_s": 43.61025288946605, "output_per_gpu": 343.3815582811865, "total_per_gpu": null, "tpot_error_pct": 2.8, "tpot_ms": 22.930387552087495, "tpot_relative": 3.011612, "ttft_error_pct": 41.03, "ttft_ms": 375.7590541469535, "ttft_relative": 1.58751, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21148.28484079059, "interactivity_tok_s": 44.668514961717165, "output_per_gpu": 342.2984248361667, "total_per_gpu": 3060.9415050947036, "tpot_error_pct": 5.11, "tpot_ms": 22.387133327737512, "tpot_relative": 2.940263, "ttft_error_pct": 32.33, "ttft_ms": 431.1954670656243, "ttft_relative": 1.821718, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 22472.380074547254, "interactivity_tok_s": 42.38748114061135, "output_per_gpu": 322.21532748666846, "total_per_gpu": 2881.3520539969977, "tpot_ms": 23.591871304707045, "tpot_relative": 3.098489, "ttft_ms": 637.1657464798773, "ttft_relative": 2.691904, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35983.950788817856, "interactivity_tok_s": 28.75687953209812, "output_per_gpu": 454.86945266405866, "total_per_gpu": null, "tpot_error_pct": 1.92, "tpot_ms": 34.77428762337759, "tpot_relative": 4.567156, "ttft_error_pct": 56.86, "ttft_ms": 409.8545501025785, "ttft_relative": 1.731557, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32245.790890870478, "interactivity_tok_s": 29.204731828318657, "output_per_gpu": 449.948427981536, "total_per_gpu": 4055.7352304393703, "tpot_error_pct": 3.42, "tpot_ms": 34.24102662125252, "tpot_relative": 4.497119, "ttft_error_pct": 28.47, "ttft_ms": 679.5253783015622, "ttft_relative": 2.870865, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 33632.93342101824, "interactivity_tok_s": 28.205196879051233, "output_per_gpu": 431.1346330016425, "total_per_gpu": 3886.1518596061496, "tpot_ms": 35.454459129931735, "tpot_relative": 4.656488, "ttft_ms": 949.9685740258428, "ttft_relative": 4.013436, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 55928.54844575075, "interactivity_tok_s": 18.440515303276232, "output_per_gpu": 585.3182481886345, "total_per_gpu": null, "tpot_error_pct": 0.63, "tpot_ms": 54.22841951831656, "tpot_relative": 7.122207, "ttft_error_pct": 70.88, "ttft_ms": 452.8752785129139, "ttft_relative": 1.913312, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 50359.94907362685, "interactivity_tok_s": 18.717333140313134, "output_per_gpu": 576.4496582523518, "total_per_gpu": 5204.234694320973, "tpot_error_pct": 2.1, "tpot_ms": 53.42641457004437, "tpot_relative": 7.016874, "ttft_error_pct": 24.16, "ttft_ms": 1179.7263736007822, "ttft_relative": 4.984119, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 51788.12752067388, "interactivity_tok_s": 18.324538670860726, "output_per_gpu": 560.1095046383984, "total_per_gpu": 5056.714450131595, "tpot_ms": 54.5716330414461, "tpot_relative": 7.167284, "ttft_ms": 1555.4502009385033, "ttft_relative": 6.57148, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 90722.8252952924, "interactivity_tok_s": 11.339938547592952, "output_per_gpu": 721.670646685618, "total_per_gpu": null, "tpot_error_pct": 2.7, "tpot_ms": 88.18389939267024, "tpot_relative": 11.581823, "ttft_error_pct": 81.54, "ttft_ms": 510.6962165907324, "ttft_relative": 2.157594, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 81685.19046768939, "interactivity_tok_s": 11.608948784929021, "output_per_gpu": 714.2320643895478, "total_per_gpu": 6426.240405616801, "tpot_error_pct": 4.95, "tpot_ms": 86.14044376681382, "tpot_relative": 11.313442, "ttft_error_pct": 21.14, "ttft_ms": 2181.4923247828074, "ttft_relative": 9.216388, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26986223642", "measured": {"e2e_ms": 86411.83020444878, "interactivity_tok_s": 11.03377373683001, "output_per_gpu": 675.0418664333316, "total_per_gpu": 6073.6300340485495, "tpot_ms": 90.6308234926067, "tpot_relative": 11.903195, "ttft_ms": 2766.1818072014285, "ttft_relative": 11.68659, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.99, "tpot_shape_error_pct": 7.12, "ttft_mape_pct": 31.63, "ttft_shape_error_pct": 36.95}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.2, "tpot_shape_error_pct": 7.05, "ttft_mape_pct": 28.92, "ttft_shape_error_pct": 19.01}, "framework": "vllm", "id": "44c332f10bbb0f53", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9661.38321437328, "interactivity_tok_s": 108.44091044045345, "output_per_gpu": 52.94272969516822, "total_per_gpu": null, "tpot_error_pct": 8.97, "tpot_ms": 9.221611990698982, "tpot_relative": 1.089698, "ttft_error_pct": 2.34, "ttft_ms": 227.67414788822126, "ttft_relative": 0.976634, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8451.754741224993, "interactivity_tok_s": 110.62349281447216, "output_per_gpu": 53.205350031511514, "total_per_gpu": 478.2843658382215, "tpot_error_pct": 6.82, "tpot_ms": 9.039671181573617, "tpot_relative": 1.068199, "ttft_error_pct": 38.64, "ttft_ms": 143.03897279999993, "ttft_relative": 0.613582, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 8010.319476737641, "interactivity_tok_s": 118.16785332401207, "output_per_gpu": 56.07343520366675, "total_per_gpu": 504.06674104901947, "tpot_ms": 8.46253843046497, "tpot_relative": 1.0, "ttft_ms": 233.12121599446982, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11215.545856384198, "interactivity_tok_s": 93.29811054038792, "output_per_gpu": 91.21268042586443, "total_per_gpu": null, "tpot_error_pct": 6.91, "tpot_ms": 10.718330673664703, "tpot_relative": 1.266562, "ttft_error_pct": 6.29, "ttft_ms": 250.6935772252077, "ttft_relative": 1.075379, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9881.691645575007, "interactivity_tok_s": 95.37848613740626, "output_per_gpu": 92.15130042991078, "total_per_gpu": 819.7284224798331, "tpot_error_pct": 4.58, "tpot_ms": 10.484544686097848, "tpot_relative": 1.238936, "ttft_error_pct": 25.38, "ttft_ms": 175.98814764999892, "ttft_relative": 0.754921, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 9516.707298799884, "interactivity_tok_s": 99.74663343065818, "output_per_gpu": 95.67243811458232, "total_per_gpu": 851.0505701448619, "tpot_ms": 10.025401014613486, "tpot_relative": 1.18468, "ttft_ms": 235.861168889096, "ttft_relative": 1.011753, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14056.727203038612, "interactivity_tok_s": 74.2272652111493, "output_per_gpu": 145.55308433087615, "total_per_gpu": null, "tpot_error_pct": 5.22, "tpot_ms": 13.47213853501631, "tpot_relative": 1.591974, "ttft_error_pct": 11.74, "ttft_ms": 274.7294817169271, "ttft_relative": 1.178483, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12274.311879987605, "interactivity_tok_s": 75.93749374129675, "output_per_gpu": 144.78396168774125, "total_per_gpu": 1306.639015229257, "tpot_error_pct": 2.85, "tpot_ms": 13.168725365190378, "tpot_relative": 1.55612, "ttft_error_pct": 26.73, "ttft_ms": 228.0608529437506, "ttft_relative": 0.978293, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 12024.89496593189, "interactivity_tok_s": 78.09883947070426, "output_per_gpu": 147.65347270916197, "total_per_gpu": 1332.535633967354, "tpot_ms": 12.80428757683539, "tpot_relative": 1.513055, "ttft_ms": 311.26700362947304, "ttft_relative": 1.335215, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18380.353605988723, "interactivity_tok_s": 56.57946208923551, "output_per_gpu": 222.6290139851682, "total_per_gpu": null, "tpot_error_pct": 1.38, "tpot_ms": 17.674257815014723, "tpot_relative": 2.088529, "ttft_error_pct": 41.03, "ttft_ms": 299.5878612286599, "ttft_relative": 1.285116, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16328.51226815003, "interactivity_tok_s": 57.85332893172946, "output_per_gpu": 221.14790853855857, "total_per_gpu": 1977.5750131907675, "tpot_error_pct": 0.85, "tpot_ms": 17.28509004520833, "tpot_relative": 2.042542, "ttft_error_pct": 34.44, "ttft_ms": 333.0273898406252, "ttft_relative": 1.428559, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 16641.970873516402, "interactivity_tok_s": 57.358888390106266, "output_per_gpu": 216.95092361975273, "total_per_gpu": 1940.0442376974881, "tpot_ms": 17.434089607853842, "tpot_relative": 2.060149, "ttft_ms": 507.9967936617322, "ttft_relative": 2.17911, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26197.545675983372, "interactivity_tok_s": 39.54275940930221, "output_per_gpu": 312.3956763439367, "total_per_gpu": null, "tpot_error_pct": 1.85, "tpot_ms": 25.289079845165176, "tpot_relative": 2.988356, "ttft_error_pct": 57.41, "ttft_ms": 326.8169943793997, "ttft_relative": 1.401919, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23485.939630376535, "interactivity_tok_s": 40.18639219869936, "output_per_gpu": 308.3101685412963, "total_per_gpu": 2779.0394069494287, "tpot_error_pct": 3.42, "tpot_ms": 24.88404520255404, "tpot_relative": 2.940494, "ttft_error_pct": 28.99, "ttft_ms": 544.8746264078129, "ttft_relative": 2.337302, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 24521.360154853028, "interactivity_tok_s": 38.81067252022748, "output_per_gpu": 295.43963720778214, "total_per_gpu": 2663.0272950770222, "tpot_ms": 25.766108522824915, "tpot_relative": 3.044726, "ttft_ms": 767.297154784319, "ttft_relative": 3.291409, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 39754.64056434534, "interactivity_tok_s": 25.96720247637783, "output_per_gpu": 411.72551852172785, "total_per_gpu": null, "tpot_error_pct": 5.59, "tpot_ms": 38.510116787116075, "tpot_relative": 4.550658, "ttft_error_pct": 70.96, "ttft_ms": 358.7910911255982, "ttft_relative": 1.539075, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 36031.286737100934, "interactivity_tok_s": 26.272772094957197, "output_per_gpu": 402.2829783327348, "total_per_gpu": 3631.8436533056088, "tpot_error_pct": 6.69, "tpot_ms": 38.06221880149222, "tpot_relative": 4.497731, "ttft_error_pct": 19.35, "ttft_ms": 996.5444940562509, "ttft_relative": 4.274791, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956500512", "measured": {"e2e_ms": 38777.5204030062, "interactivity_tok_s": 24.51556529683783, "output_per_gpu": 373.6939471238579, "total_per_gpu": 3373.7395396778247, "tpot_ms": 40.790411638151625, "tpot_relative": 4.820115, "ttft_ms": 1235.5715697267442, "ttft_relative": 5.300125, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 106}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 261, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 261, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "frameworks": ["dynamo-vllm", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "hf_model_paths": ["MiniMaxAI/MiniMax-M3-MXFP8"], "model": "MiniMaxAI/MiniMax-M3-MXFP8", "precisions": ["fp8"], "rows": 261, "workloads": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 87, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 87, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu_skus": ["b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 14, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 14, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "b300", "precisions": ["fp8"], "rows": 14, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5ca7275e893c31c1", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 6414.3206520646345, "interactivity_tok_s": 154.7761977430471, "output_per_gpu": 34.99436562186492, "total_per_gpu": 51.4899232794284, "tpot_ms": 6.460941763540139, "tpot_relative": 1.0, "ttft_ms": 489.75278608850203, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "0ed1d2227ded5303", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 47320.705785888946, "interactivity_tok_s": 26.85655179311431, "output_per_gpu": 4799.538417378889, "total_per_gpu": 7748.778583392096, "tpot_ms": 37.23486200698288, "tpot_relative": 1.0, "ttft_ms": 13008.527926866742, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cc796c9204a48385", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 6473.636957159033, "interactivity_tok_s": 144.97259162447264, "output_per_gpu": 69.57842681407784, "total_per_gpu": 112.61374722432234, "tpot_ms": 6.897855579421064, "tpot_relative": 1.0, "ttft_ms": 161.54447264270857, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 8757.432346743008, "interactivity_tok_s": 107.33801718321352, "output_per_gpu": 204.60756449032257, "total_per_gpu": 330.0334843876257, "tpot_ms": 9.316363635571134, "tpot_relative": 1.350617, "ttft_ms": 187.19252316077473, "ttft_relative": 1.158768, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 13262.740845578446, "interactivity_tok_s": 70.90742427677668, "output_per_gpu": 539.0481726032847, "total_per_gpu": 872.4542646823311, "tpot_ms": 14.102895574046626, "tpot_relative": 2.044533, "ttft_ms": 304.3045656489994, "ttft_relative": 1.88372, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 16264.040681444023, "interactivity_tok_s": 57.990928545055645, "output_per_gpu": 877.2913187662087, "total_per_gpu": 1419.5889220493098, "tpot_ms": 17.244076359685412, "tpot_relative": 2.499918, "ttft_ms": 423.0543777504863, "ttft_relative": 2.618811, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 118561.22877697059, "interactivity_tok_s": 15.370390354464195, "output_per_gpu": 3862.308906040643, "total_per_gpu": 6239.76441370516, "tpot_ms": 65.06015637459453, "tpot_relative": 9.431939, "ttft_ms": 58682.95239594399, "ttft_relative": 363.261902, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "c3bec4267017b2c9", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 10554.458250632524, "interactivity_tok_s": 89.18401147970673, "output_per_gpu": 342.60929058327156, "total_per_gpu": 459.5856654959006, "tpot_ms": 11.212772148374867, "tpot_relative": 1.0, "ttft_ms": 182.36952968327387, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "fcc3c978d2832882", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 21775.25986005985, "interactivity_tok_s": 43.53761235467965, "output_per_gpu": 2607.516650367528, "total_per_gpu": 3508.048142609337, "tpot_ms": 22.968645865406874, "tpot_relative": 1.0, "ttft_ms": 611.0280170445094, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 59762.55387291405, "interactivity_tok_s": 19.404650626530607, "output_per_gpu": 7453.7146166446155, "total_per_gpu": 10034.891983504227, "tpot_ms": 51.534037857541776, "tpot_relative": 2.243669, "ttft_ms": 12334.208704322788, "ttft_relative": 20.185995, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "d4da17bc763d82d1", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27797429264", "measured": {"e2e_ms": 15830.011387930426, "interactivity_tok_s": 58.28061551185021, "output_per_gpu": 453.9497746899257, "total_per_gpu": 912.7319988896627, "tpot_ms": 17.158363740970955, "tpot_relative": 1.0, "ttft_ms": 126.80692101712337, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27797429264", "measured": {"e2e_ms": 21263.757855167933, "interactivity_tok_s": 43.73842884739337, "output_per_gpu": 680.9653245365615, "total_per_gpu": 1359.7765776732642, "tpot_ms": 22.863189793329667, "tpot_relative": 1.332481, "ttft_ms": 166.3785152035416, "ttft_relative": 1.312062, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27797429264", "measured": {"e2e_ms": 28184.81077150641, "interactivity_tok_s": 32.95495584483631, "output_per_gpu": 1024.1795383980007, "total_per_gpu": 2048.820856061434, "tpot_ms": 30.344449700019528, "tpot_relative": 1.768493, "ttft_ms": 236.6945787804525, "ttft_relative": 1.866575, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27797429264", "measured": {"e2e_ms": 42626.57315310753, "interactivity_tok_s": 21.809781162504013, "output_per_gpu": 1352.9548934109735, "total_per_gpu": 2708.8820111230934, "tpot_ms": 45.85098734136902, "tpot_relative": 2.672224, "ttft_ms": 464.5728599578433, "ttft_relative": 3.663624, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "6e8fd3129cb854bf", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 16112.312406266576, "interactivity_tok_s": 58.22286394171799, "output_per_gpu": 884.9110516911619, "total_per_gpu": 1193.2656605509203, "tpot_ms": 17.175383213732253, "tpot_relative": 1.0, "ttft_ms": 334.3023647646987, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 19367.075939503957, "interactivity_tok_s": 48.727706286440615, "output_per_gpu": 1478.1311894791509, "total_per_gpu": 1988.778907331748, "tpot_ms": 20.5222054599001, "tpot_relative": 1.194862, "ttft_ms": 460.3263422074633, "ttft_relative": 1.376976, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "4d1442a567afc87e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 9568.475541502266, "interactivity_tok_s": 98.69466223508509, "output_per_gpu": 188.6056338006496, "total_per_gpu": 303.60103397889577, "tpot_ms": 10.13226021907909, "tpot_relative": 1.0, "ttft_ms": 194.61682917899452, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "4f3c1bae0ef4c7f4", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 24944.19691010321, "interactivity_tok_s": 40.669058248002386, "output_per_gpu": 2001.157595600851, "total_per_gpu": 3232.3774608541407, "tpot_ms": 24.5887178872434, "tpot_relative": 1.0, "ttft_ms": 2299.4907793266293, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 48558.95985910118, "interactivity_tok_s": 28.83051816910197, "output_per_gpu": 4669.266177185712, "total_per_gpu": 7543.446585785324, "tpot_ms": 34.685467466613645, "tpot_relative": 1.410625, "ttft_ms": 16635.101507473337, "ttft_relative": 7.234254, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ea5ce241fd9e0439", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 32728.83535194637, "interactivity_tok_s": 33.27374938321497, "output_per_gpu": 3495.4498803934675, "total_per_gpu": 4705.032324693348, "tpot_ms": 30.05372158343095, "tpot_relative": 1.0, "ttft_ms": 5053.443921617372, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 72517.96744667403, "interactivity_tok_s": 17.82227235000602, "output_per_gpu": 6342.30663647187, "total_per_gpu": 8538.60998127502, "tpot_ms": 56.10956786886169, "tpot_relative": 1.866976, "ttft_ms": 20884.408679821412, "ttft_relative": 4.132708, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 12, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 12, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 12, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "c8e2d34d5a5902eb", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 84988.94056726406, "interactivity_tok_s": 24.33622713466777, "output_per_gpu": 5311.79098060003, "total_per_gpu": 8580.33883199853, "tpot_ms": 41.091003731448026, "tpot_relative": 1.0, "ttft_ms": 47167.734382348026, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "65304328094d9080", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 10452.180072632655, "interactivity_tok_s": 90.05971392580736, "output_per_gpu": 345.7786647500927, "total_per_gpu": 463.83715246866836, "tpot_ms": 11.103743909555565, "tpot_relative": 1.0, "ttft_ms": 181.0091625979112, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e05a7deca79b89cd", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 170642.69569098967, "interactivity_tok_s": 11.14906204908718, "output_per_gpu": 10656.533394438375, "total_per_gpu": 14344.921143787777, "tpot_ms": 89.69364378789821, "tpot_relative": 1.0, "ttft_ms": 88087.06535732857, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "bad05e0647c35362", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 86220.40235830261, "interactivity_tok_s": 18.466826171951695, "output_per_gpu": 6850.1500745323565, "total_per_gpu": 10373.72721322848, "tpot_ms": 54.15115681972726, "tpot_relative": 1.0, "ttft_ms": 36378.57000219006, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "19db185dc754e8dd", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 6632.147050095955, "interactivity_tok_s": 141.2754981055541, "output_per_gpu": 34.82841151300925, "total_per_gpu": 46.299674035151185, "tpot_ms": 7.07836824792399, "tpot_relative": 1.0, "ttft_ms": 98.69174669147469, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 7875.207610319194, "interactivity_tok_s": 118.62909101004574, "output_per_gpu": 114.34438494880355, "total_per_gpu": 154.2234455510355, "tpot_ms": 8.42963552603904, "tpot_relative": 1.190901, "ttft_ms": 162.05973842734238, "ttft_relative": 1.64208, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 9175.25165877596, "interactivity_tok_s": 101.72018618859498, "output_per_gpu": 195.4535982831967, "total_per_gpu": 263.66308167231017, "tpot_ms": 9.830890381442513, "tpot_relative": 1.388864, "ttft_ms": 154.31372953462414, "ttft_relative": 1.563593, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 11391.690617443965, "interactivity_tok_s": 82.05369793416607, "output_per_gpu": 314.7322740725447, "total_per_gpu": 423.0545325476633, "tpot_ms": 12.187141167023666, "tpot_relative": 1.721744, "ttft_ms": 181.83011682012875, "ttft_relative": 1.842404, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f7489291d2a1bf5f", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 22834.098182029902, "interactivity_tok_s": 41.38931015699291, "output_per_gpu": 2490.37391594449, "total_per_gpu": 3352.154994644928, "tpot_ms": 24.16082790959601, "tpot_relative": 1.0, "ttft_ms": 583.9539696616043, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 39670.36677804794, "interactivity_tok_s": 27.256025810478143, "output_per_gpu": 5683.560897733575, "total_per_gpu": 7651.744482283289, "tpot_ms": 36.689134613879254, "tpot_relative": 1.518538, "ttft_ms": 5903.761729576411, "ttft_relative": 10.109978, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "2692a6ae93c8417b", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 44153.09965871216, "interactivity_tok_s": 24.523633757009684, "output_per_gpu": 5151.819958724464, "total_per_gpu": 8317.531539513597, "tpot_ms": 40.77699128556616, "tpot_relative": 1.0, "ttft_ms": 6576.913938591022, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "9a006bdde8c4a3e4", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 5963.711011626583, "interactivity_tok_s": 157.81767367707656, "output_per_gpu": 37.52906720917794, "total_per_gpu": 55.21942624219814, "tpot_ms": 6.336425931902789, "tpot_relative": 1.0, "ttft_ms": 165.5352510031662, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 17, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 17, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "h100", "precisions": ["fp8"], "rows": 17, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "f3ab60f11137a393", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8261.031255312264, "interactivity_tok_s": 112.84688035508685, "output_per_gpu": 13.940149612805756, "total_per_gpu": 27.781958686775788, "tpot_ms": 8.861565307373803, "tpot_relative": 1.0, "ttft_ms": 104.42492477595806, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9320.549975568429, "interactivity_tok_s": 101.74772283649843, "output_per_gpu": 25.01361812407288, "total_per_gpu": 49.56989022310257, "tpot_ms": 9.82822978364765, "tpot_relative": 1.109085, "ttft_ms": 139.03788845054805, "ttft_relative": 1.331463, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10678.894262993708, "interactivity_tok_s": 87.08938217570008, "output_per_gpu": 41.755237018235064, "total_per_gpu": 83.91882430723605, "tpot_ms": 11.4824560126346, "tpot_relative": 1.295759, "ttft_ms": 151.19705016259104, "ttft_relative": 1.447902, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 13122.595867048949, "interactivity_tok_s": 71.70763250959278, "output_per_gpu": 69.4404044498421, "total_per_gpu": 138.3678957596933, "tpot_ms": 13.945516885754436, "tpot_relative": 1.573708, "ttft_ms": 175.71843500481918, "ttft_relative": 1.682725, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 16724.97995685553, "interactivity_tok_s": 55.32939535455635, "output_per_gpu": 107.38408772462762, "total_per_gpu": 215.91131552998525, "tpot_ms": 18.073575422104273, "tpot_relative": 2.039547, "ttft_ms": 183.29348156694323, "ttft_relative": 1.755266, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 21548.394809191814, "interactivity_tok_s": 43.41608246612743, "output_per_gpu": 167.7478980426535, "total_per_gpu": 334.9651656162651, "tpot_ms": 23.03293948228021, "tpot_relative": 2.599195, "ttft_ms": 287.60653720528353, "ttft_relative": 2.754194, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 29462.35779044946, "interactivity_tok_s": 31.731069304857257, "output_per_gpu": 244.79432717739866, "total_per_gpu": 489.699026550636, "tpot_ms": 31.51485348295288, "tpot_relative": 3.556353, "ttft_ms": 420.8557949299575, "ttft_relative": 4.030224, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 39801.405725877885, "interactivity_tok_s": 23.502432828540925, "output_per_gpu": 361.5670306281129, "total_per_gpu": 723.9283658706366, "tpot_ms": 42.54878664244573, "tpot_relative": 4.801498, "ttft_ms": 671.0679790068752, "ttft_relative": 6.42632, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 69359.1827907564, "interactivity_tok_s": 14.52824830712546, "output_per_gpu": 413.4405229717141, "total_per_gpu": 826.4814570189901, "tpot_ms": 68.83142267809014, "tpot_relative": 7.767411, "ttft_ms": 5849.973309347297, "ttft_relative": 56.020853, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "0fab9e2bd73cfb4f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 6487.291677016765, "interactivity_tok_s": 144.1965768314339, "output_per_gpu": 17.75175898055267, "total_per_gpu": 35.37828849141401, "tpot_ms": 6.934977389712947, "tpot_relative": 1.0, "ttft_ms": 104.38444204628468, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7090.010438440368, "interactivity_tok_s": 134.3200634873843, "output_per_gpu": 32.87279666197184, "total_per_gpu": 65.14455101127892, "tpot_ms": 7.444904164253337, "tpot_relative": 1.07353, "ttft_ms": 133.89686750015244, "ttft_relative": 1.282728, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8349.333583726548, "interactivity_tok_s": 111.68171967459463, "output_per_gpu": 53.41508681686159, "total_per_gpu": 107.35255278235849, "tpot_ms": 8.954016851761283, "tpot_relative": 1.291139, "ttft_ms": 139.6100240526721, "ttft_relative": 1.33746, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10642.33217164874, "interactivity_tok_s": 88.44648717959122, "output_per_gpu": 85.70098472195725, "total_per_gpu": 170.76894949648891, "tpot_ms": 11.306271530823977, "tpot_relative": 1.630326, "ttft_ms": 142.7316326647997, "ttft_relative": 1.367365, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 13915.016842268233, "interactivity_tok_s": 66.73259492754157, "output_per_gpu": 129.1531833091275, "total_per_gpu": 259.6812461141179, "tpot_ms": 14.985180796367992, "tpot_relative": 2.160812, "ttft_ms": 199.9049277830636, "ttft_relative": 1.915084, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 48697.53686901531, "interactivity_tok_s": 19.220476817334756, "output_per_gpu": 74.67741199590797, "total_per_gpu": 149.11859980888073, "tpot_ms": 52.02784558903919, "tpot_relative": 7.502237, "ttft_ms": 677.9403139575152, "ttft_relative": 6.494649, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 27450.170171799255, "interactivity_tok_s": 34.04358437344221, "output_per_gpu": 263.43212955894643, "total_per_gpu": 526.9830346750275, "tpot_ms": 29.374110229712223, "tpot_relative": 4.235646, "ttft_ms": 380.52143427485134, "ttft_relative": 3.645385, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 36420.4997234021, "interactivity_tok_s": 25.66792111297697, "output_per_gpu": 395.7095940493426, "total_per_gpu": 792.2884984336846, "tpot_ms": 38.959134851572706, "tpot_relative": 5.617774, "ttft_ms": 590.1066031045048, "ttft_relative": 5.653205, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 37, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 37, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "h200", "precisions": ["fp8"], "rows": 37, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "109b4c4f784dd356", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 6367.686954047531, "interactivity_tok_s": 147.24975157496243, "output_per_gpu": 18.08567338108829, "total_per_gpu": 36.04376169925378, "tpot_ms": 6.79118293446435, "tpot_relative": 1.0, "ttft_ms": 117.06668231636286, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 6964.514001645148, "interactivity_tok_s": 137.7318411982585, "output_per_gpu": 33.46507508439919, "total_per_gpu": 66.31827870775243, "tpot_ms": 7.260485239288619, "tpot_relative": 1.069105, "ttft_ms": 178.10722747817636, "ttft_relative": 1.521417, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7918.9226383809, "interactivity_tok_s": 118.51578729667824, "output_per_gpu": 56.2987678139006, "total_per_gpu": 113.14811607524435, "tpot_ms": 8.43769444400449, "tpot_relative": 1.242448, "ttft_ms": 179.44837831892073, "ttft_relative": 1.532873, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9638.471564010251, "interactivity_tok_s": 98.00304163065442, "output_per_gpu": 94.54566214212947, "total_per_gpu": 188.39297419793616, "tpot_ms": 10.203764937916064, "tpot_relative": 1.502502, "ttft_ms": 164.6232504514046, "ttft_relative": 1.406235, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 12761.072010087082, "interactivity_tok_s": 73.10988820249212, "output_per_gpu": 140.8693484194178, "total_per_gpu": 283.23829889102706, "tpot_ms": 13.678040338815793, "tpot_relative": 2.014088, "ttft_ms": 243.18233298836276, "ttft_relative": 2.077298, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 17232.694675651146, "interactivity_tok_s": 54.819584567135365, "output_per_gpu": 210.22261462918965, "total_per_gpu": 419.78024015326673, "tpot_ms": 18.24165593183837, "tpot_relative": 2.686079, "ttft_ms": 392.11880184011534, "ttft_relative": 3.349534, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 24935.945681945304, "interactivity_tok_s": 37.6305908300269, "output_per_gpu": 290.0522958070772, "total_per_gpu": 580.2353695989443, "tpot_ms": 26.57412434784472, "tpot_relative": 3.913033, "ttft_ms": 441.53524505964015, "ttft_relative": 3.771656, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 34073.663671908434, "interactivity_tok_s": 27.565270600477596, "output_per_gpu": 423.5232435546529, "total_per_gpu": 847.976899558918, "tpot_ms": 36.277532497093425, "tpot_relative": 5.341858, "ttft_ms": 702.086555439746, "ttft_relative": 5.997322, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "3a80177ea63b5518", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 6697.105513885617, "interactivity_tok_s": 139.92602881686932, "output_per_gpu": 34.39267232972666, "total_per_gpu": 68.54272216092583, "tpot_ms": 7.146633177939808, "tpot_relative": 1.0, "ttft_ms": 119.24556791782379, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7619.683904666454, "interactivity_tok_s": 124.89948278166506, "output_per_gpu": 61.19411186223055, "total_per_gpu": 121.26935784598568, "tpot_ms": 8.006438279236795, "tpot_relative": 1.120309, "ttft_ms": 138.76356901600957, "ttft_relative": 1.163679, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9033.884038310498, "interactivity_tok_s": 103.27443222401375, "output_per_gpu": 98.83133742056175, "total_per_gpu": 198.62920757516645, "tpot_ms": 9.682938733866758, "tpot_relative": 1.354895, "ttft_ms": 155.41080590337515, "ttft_relative": 1.303284, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 11799.093106365763, "interactivity_tok_s": 80.37622198164685, "output_per_gpu": 154.73079832873535, "total_per_gpu": 308.3186963496043, "tpot_ms": 12.441490472497458, "tpot_relative": 1.740888, "ttft_ms": 246.85564888641238, "ttft_relative": 2.070145, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 15719.307494757231, "interactivity_tok_s": 59.130932543148255, "output_per_gpu": 228.72032429999177, "total_per_gpu": 459.8754541240139, "tpot_ms": 16.911622343691146, "tpot_relative": 2.366376, "ttft_ms": 244.3653394584544, "ttft_relative": 2.049261, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 21395.106902805855, "interactivity_tok_s": 43.872807463483795, "output_per_gpu": 338.7713270686997, "total_per_gpu": 676.4710318382353, "tpot_ms": 22.7931618197062, "tpot_relative": 3.189357, "ttft_ms": 358.1229796342086, "ttft_relative": 3.003239, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 31610.058911525994, "interactivity_tok_s": 29.589054559666447, "output_per_gpu": 457.8603927945216, "total_per_gpu": 915.92722442904, "tpot_ms": 33.796280918117745, "tpot_relative": 4.728979, "ttft_ms": 466.6879034397425, "ttft_relative": 3.913671, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "46a43296bb63c76a", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 70412.46195588137, "interactivity_tok_s": 13.307526554299079, "output_per_gpu": 411.5015779967548, "total_per_gpu": 822.6054410531008, "tpot_ms": 75.14544464139685, "tpot_relative": 1.0, "ttft_ms": 1056.2780052838207, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 104779.71276361823, "interactivity_tok_s": 8.9679824194485, "output_per_gpu": 555.2788428061596, "total_per_gpu": 1110.0461743754793, "tpot_ms": 111.50780111157896, "tpot_relative": 1.483893, "ttft_ms": 1828.5029293339903, "ttft_relative": 1.731081, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 178727.2517360672, "interactivity_tok_s": 5.262756330530494, "output_per_gpu": 653.3037351429226, "total_per_gpu": 1306.5689943464647, "tpot_ms": 190.01449757397344, "tpot_relative": 2.528623, "ttft_ms": 3362.796347127005, "ttft_relative": 3.183628, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "1db34992706ef9a5", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7697.948662564158, "interactivity_tok_s": 121.3457279316461, "output_per_gpu": 14.96076041847981, "total_per_gpu": 29.81598041725035, "tpot_ms": 8.240916405094202, "tpot_relative": 1.0, "ttft_ms": 113.0285793915391, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8504.093184880912, "interactivity_tok_s": 112.33954775078814, "output_per_gpu": 27.412734396481422, "total_per_gpu": 54.324257610733675, "tpot_ms": 8.901584704777168, "tpot_relative": 1.080169, "ttft_ms": 184.5229591242969, "ttft_relative": 1.632534, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9401.636824174784, "interactivity_tok_s": 99.15802391443442, "output_per_gpu": 47.38492251055539, "total_per_gpu": 95.23325146588698, "tpot_ms": 10.084912551937515, "tpot_relative": 1.223761, "ttft_ms": 155.14303853269666, "ttft_relative": 1.3726, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 11562.013713573106, "interactivity_tok_s": 81.64042689338928, "output_per_gpu": 78.82248563397387, "total_per_gpu": 157.06275852122315, "tpot_ms": 12.24883355039112, "tpot_relative": 1.486344, "ttft_ms": 186.69982120627537, "ttft_relative": 1.651793, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 14377.172590332339, "interactivity_tok_s": 64.59576737226806, "output_per_gpu": 124.93088946324691, "total_per_gpu": 251.19171067050524, "tpot_ms": 15.480890477497681, "tpot_relative": 1.87854, "ttft_ms": 208.29409080324695, "ttft_relative": 1.842844, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 17975.820725598896, "interactivity_tok_s": 52.04899063828727, "output_per_gpu": 200.9834785679437, "total_per_gpu": 401.3311938342509, "tpot_ms": 19.212668444417428, "tpot_relative": 2.331375, "ttft_ms": 240.91714317910373, "ttft_relative": 2.131471, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 26793.787054622953, "interactivity_tok_s": 34.87959167166107, "output_per_gpu": 269.5777300125805, "total_per_gpu": 539.2770064937976, "tpot_ms": 28.670060401322843, "tpot_relative": 3.478989, "ttft_ms": 376.98405671617365, "ttft_relative": 3.335299, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 39410.23500973097, "interactivity_tok_s": 23.740282584411634, "output_per_gpu": 366.15517275643396, "total_per_gpu": 733.1147295376105, "tpot_ms": 42.122497760688866, "tpot_relative": 5.111385, "ttft_ms": 668.0122604113421, "ttft_relative": 5.910118, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 58383.81584349627, "interactivity_tok_s": 16.124530973602575, "output_per_gpu": 496.42773519379824, "total_per_gpu": 992.3756745917188, "tpot_ms": 62.01730776771723, "tpot_relative": 7.525535, "ttft_ms": 1194.1479911576607, "ttft_relative": 10.565009, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27447149961", "measured": {"e2e_ms": 84808.00379424072, "interactivity_tok_s": 11.185049069000323, "output_per_gpu": 683.9769264277256, "total_per_gpu": 1367.3237876402181, "tpot_ms": 89.40506150943298, "tpot_relative": 10.848922, "ttft_ms": 2376.6717451606382, "ttft_relative": 21.027175, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "90a3b316bd504ac9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7879.5135064050555, "interactivity_tok_s": 118.82264707482473, "output_per_gpu": 29.231521204905473, "total_per_gpu": 58.25682916059032, "tpot_ms": 8.415904077362308, "tpot_relative": 1.0, "ttft_ms": 133.5341079160571, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9037.202467955649, "interactivity_tok_s": 105.59078093472863, "output_per_gpu": 51.59446533835825, "total_per_gpu": 102.245583595951, "tpot_ms": 9.470523763037173, "tpot_relative": 1.125313, "ttft_ms": 186.86422053724527, "ttft_relative": 1.399374, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10582.786758663133, "interactivity_tok_s": 88.23342798108938, "output_per_gpu": 84.35750782421513, "total_per_gpu": 169.53999985690484, "tpot_ms": 11.333573033276288, "tpot_relative": 1.346685, "ttft_ms": 188.44137405976653, "ttft_relative": 1.411185, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 13462.512190127745, "interactivity_tok_s": 69.99066586312526, "output_per_gpu": 135.49361108664772, "total_per_gpu": 269.9864150198556, "tpot_ms": 14.287619465652952, "tpot_relative": 1.697693, "ttft_ms": 196.4300490450114, "ttft_relative": 1.47101, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 17545.292865278316, "interactivity_tok_s": 52.86796221396884, "output_per_gpu": 204.86492396578893, "total_per_gpu": 411.91070461792, "tpot_ms": 18.915047187799093, "tpot_relative": 2.247536, "ttft_ms": 235.05327025195584, "ttft_relative": 1.760249, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 23675.01331618696, "interactivity_tok_s": 39.63470868878692, "output_per_gpu": 305.94038570684984, "total_per_gpu": 610.9130019676396, "tpot_ms": 25.230411250200778, "tpot_relative": 2.997944, "ttft_ms": 387.2652546619065, "ttft_relative": 2.900122, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 33782.800294196204, "interactivity_tok_s": 27.699052885956593, "output_per_gpu": 428.0561775933575, "total_per_gpu": 856.3053559837845, "tpot_ms": 36.1023174372507, "tpot_relative": 4.289773, "ttft_ms": 512.4275281974406, "ttft_relative": 3.837428, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 51165.37124850147, "interactivity_tok_s": 18.290550693834792, "output_per_gpu": 564.6775727590908, "total_per_gpu": 1130.595651326794, "tpot_ms": 54.673039469340345, "tpot_relative": 6.496395, "ttft_ms": 886.5133408253314, "ttft_relative": 6.638853, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 78311.86941670785, "interactivity_tok_s": 12.071699831049473, "output_per_gpu": 740.1301822466194, "total_per_gpu": 1479.5450310727451, "tpot_ms": 82.83837520776586, "tpot_relative": 9.843075, "ttft_ms": 1917.5147461126471, "ttft_relative": 14.359738, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 87}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 174, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 174, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 30, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 30, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "b200", "precisions": ["fp8"], "rows": 30, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "d6abc84711c5cd29", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 33594.77603039913, "interactivity_tok_s": 28.11528150447276, "output_per_gpu": 431.6662513609623, "total_per_gpu": 3890.9437494649806, "tpot_ms": 35.567845900490575, "tpot_relative": 1.0, "ttft_ms": 792.3006045939474, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 50858.06496420814, "interactivity_tok_s": 18.544866918597428, "output_per_gpu": 570.7082190517842, "total_per_gpu": 5152.400511309198, "tpot_ms": 53.92327722757426, "tpot_relative": 1.516068, "ttft_ms": 1189.5052796555774, "ttft_relative": 1.501331, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "e57f7ebb3a3a141a", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 26051.82659181537, "interactivity_tok_s": 36.40739391524976, "output_per_gpu": 556.9181822027496, "total_per_gpu": 5019.93684512802, "tpot_ms": 27.46694812399455, "tpot_relative": 1.0, "ttft_ms": 725.0563084371606, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 38816.136132625885, "interactivity_tok_s": 24.48329403699387, "output_per_gpu": 747.5848411448076, "total_per_gpu": 6749.257132061745, "tpot_ms": 40.84417719645959, "tpot_relative": 1.48703, "ttft_ms": 1210.7407383814461, "ttft_relative": 1.669858, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 61923.92726488765, "interactivity_tok_s": 15.453045392715138, "output_per_gpu": 941.7879078861371, "total_per_gpu": 8473.654164983434, "tpot_ms": 64.71216349829783, "tpot_relative": 2.356001, "ttft_ms": 2197.396477654479, "ttft_relative": 3.030656, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "b50dcf4bdbc1122f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4874.704535695491, "interactivity_tok_s": 198.2222140191184, "output_per_gpu": 23.852539089514543, "total_per_gpu": 213.51650491437312, "tpot_ms": 5.044843258099975, "tpot_relative": 1.0, "ttft_ms": 186.43515440053307, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5498.5009237600025, "interactivity_tok_s": 177.16199400777884, "output_per_gpu": 42.39159615265907, "total_per_gpu": 377.7713323224051, "tpot_ms": 5.644551505533923, "tpot_relative": 1.118875, "ttft_ms": 224.0225699613802, "ttft_relative": 1.201611, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 6448.509083325916, "interactivity_tok_s": 148.37968390556807, "output_per_gpu": 69.78137845471186, "total_per_gpu": 627.2929756455288, "tpot_ms": 6.739467113546493, "tpot_relative": 1.335912, "ttft_ms": 254.15539469831856, "ttft_relative": 1.363238, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 7839.918807856884, "interactivity_tok_s": 122.49291440712575, "output_per_gpu": 116.29069637393212, "total_per_gpu": 1034.4595099901935, "tpot_ms": 8.163737509553673, "tpot_relative": 1.618234, "ttft_ms": 280.5535476458317, "ttft_relative": 1.504832, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 10341.788295053266, "interactivity_tok_s": 91.40728779925014, "output_per_gpu": 172.36922507677514, "total_per_gpu": 1555.589112804455, "tpot_ms": 10.940046730149273, "tpot_relative": 2.16856, "ttft_ms": 327.78282103536185, "ttft_relative": 1.75816, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 14629.940625834934, "interactivity_tok_s": 65.26237723341588, "output_per_gpu": 247.5420615351582, "total_per_gpu": 2213.5999333690606, "tpot_ms": 15.322763932172185, "tpot_relative": 3.037312, "ttft_ms": 442.90888081995945, "ttft_relative": 2.375673, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 22145.609513346517, "interactivity_tok_s": 42.91388237220067, "output_per_gpu": 327.862369340251, "total_per_gpu": 2955.2786038917934, "tpot_ms": 23.30248266346075, "tpot_relative": 4.61907, "ttft_ms": 653.487230974406, "ttft_relative": 3.505172, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 34115.141677392014, "interactivity_tok_s": 27.844844083073355, "output_per_gpu": 425.88464478474725, "total_per_gpu": 3844.9214297167173, "tpot_ms": 35.913291416413124, "tpot_relative": 7.118812, "ttft_ms": 1048.2424259632353, "ttft_relative": 5.622558, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 59517.305859636894, "interactivity_tok_s": 16.0333396404574, "output_per_gpu": 490.8694484627477, "total_per_gpu": 4416.5548438240985, "tpot_ms": 62.370037835203746, "tpot_relative": 12.363127, "ttft_ms": 1890.9474927965675, "ttft_relative": 10.142655, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "b75c0bc5dcb86d18", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4400.2933757030405, "interactivity_tok_s": 221.5425437888258, "output_per_gpu": 26.42404682300235, "total_per_gpu": 236.53541043022074, "tpot_ms": 4.513805713782899, "tpot_relative": 1.0, "ttft_ms": 205.56830140994862, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4807.138146203943, "interactivity_tok_s": 203.93860957842944, "output_per_gpu": 48.48233468436213, "total_per_gpu": 432.04875093299324, "tpot_ms": 4.9034363922905255, "tpot_relative": 1.08632, "ttft_ms": 224.62953794165514, "ttft_relative": 1.092725, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5541.207483832841, "interactivity_tok_s": 173.36660773201407, "output_per_gpu": 81.19130852146573, "total_per_gpu": 729.8614422189795, "tpot_ms": 5.768123475922052, "tpot_relative": 1.277885, "ttft_ms": 240.56237315235194, "ttft_relative": 1.170231, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 7030.125523315655, "interactivity_tok_s": 137.04270604022662, "output_per_gpu": 129.72356198821328, "total_per_gpu": 1153.9510601691663, "tpot_ms": 7.29699543225939, "tpot_relative": 1.616595, "ttft_ms": 270.1221502800763, "ttft_relative": 1.314026, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 9306.406459959544, "interactivity_tok_s": 101.8885033855633, "output_per_gpu": 191.69453137080825, "total_per_gpu": 1729.9951650403993, "tpot_ms": 9.814650002423052, "tpot_relative": 2.174363, "ttft_ms": 320.0507387831749, "ttft_relative": 1.556907, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 13411.576194759255, "interactivity_tok_s": 71.32147081666044, "output_per_gpu": 270.35510386066653, "total_per_gpu": 2417.6014216758003, "tpot_ms": 14.021023242364256, "tpot_relative": 3.106253, "ttft_ms": 428.2751004815509, "ttft_relative": 2.083371, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 20938.90518088192, "interactivity_tok_s": 45.426006064499504, "output_per_gpu": 347.1336499985871, "total_per_gpu": 3128.9856490576576, "tpot_ms": 22.013821742992757, "tpot_relative": 4.876998, "ttft_ms": 632.7945998498762, "ttft_relative": 3.078269, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "57c1e3cfac846839", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 42605.335395321345, "interactivity_tok_s": 22.13797340622671, "output_per_gpu": 340.09330222093047, "total_per_gpu": 3070.390167443791, "tpot_ms": 45.17125310660693, "tpot_relative": 1.0, "ttft_ms": 986.2599431590297, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 69984.32376600098, "interactivity_tok_s": 13.492440007345232, "output_per_gpu": 417.4168630816414, "total_per_gpu": 3755.67164408072, "tpot_ms": 74.11557875785283, "tpot_relative": 1.640769, "ttft_ms": 1498.0160060152002, "ttft_relative": 1.518886, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "457596f0e71e1786", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4734.294046415016, "interactivity_tok_s": 204.43626456594762, "output_per_gpu": 49.11989764744287, "total_per_gpu": 439.6977960322963, "tpot_ms": 4.891500058089827, "tpot_relative": 1.0, "ttft_ms": 188.5935200145468, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5421.269862199551, "interactivity_tok_s": 179.29087812482106, "output_per_gpu": 86.02339934237769, "total_per_gpu": 766.5947293761878, "tpot_ms": 5.577528597432642, "tpot_relative": 1.140249, "ttft_ms": 209.35750275384635, "ttft_relative": 1.110099, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 6454.71574599942, "interactivity_tok_s": 147.33983604652005, "output_per_gpu": 139.37872188649544, "total_per_gpu": 1252.9315861909101, "tpot_ms": 6.787030763929091, "tpot_relative": 1.387515, "ttft_ms": 218.20807877520565, "ttft_relative": 1.157029, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 8272.330886841519, "interactivity_tok_s": 115.83650628139901, "output_per_gpu": 220.5799940678711, "total_per_gpu": 1962.161030005141, "tpot_ms": 8.632857050874122, "tpot_relative": 1.764869, "ttft_ms": 277.0799440309929, "ttft_relative": 1.469191, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 11241.11516167577, "interactivity_tok_s": 83.86646663128103, "output_per_gpu": 317.5926516587017, "total_per_gpu": 2866.194188706955, "tpot_ms": 11.923716834244377, "tpot_relative": 2.43764, "ttft_ms": 331.26599185634404, "ttft_relative": 1.756508, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 16409.253208807786, "interactivity_tok_s": 57.98111014124018, "output_per_gpu": 441.623741772058, "total_per_gpu": 3949.1401150102242, "tpot_ms": 17.246996436667583, "tpot_relative": 3.525912, "ttft_ms": 445.844228475471, "ttft_relative": 2.364049, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 25068.75652885801, "interactivity_tok_s": 37.824685522479456, "output_per_gpu": 579.2502856377458, "total_per_gpu": 5221.233467226335, "tpot_ms": 26.43776111253307, "tpot_relative": 5.404837, "ttft_ms": 690.7538231379476, "ttft_relative": 3.662659, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 55, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 55, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "b300", "precisions": ["fp8"], "rows": 55, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "58716961962f126f", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 19734.855059561483, "interactivity_tok_s": 56.78983236743659, "output_per_gpu": 715.1664945862785, "total_per_gpu": 5153.484559006911, "tpot_ms": 17.608785909595362, "tpot_relative": 1.0, "ttft_ms": 3509.1827310931876, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 37575.932141043086, "interactivity_tok_s": 56.74313865543662, "output_per_gpu": 746.2768050413734, "total_per_gpu": 5383.027475281885, "tpot_ms": 17.623276112242145, "tpot_relative": 1.000823, "ttft_ms": 21344.175076931664, "ttft_relative": 6.082378, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "54b408eaa6b4577b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27951287552", "measured": {"e2e_ms": 9457.013981679484, "interactivity_tok_s": 103.99100434426414, "output_per_gpu": 189.56699495113605, "total_per_gpu": 1237.8708433490128, "tpot_ms": 9.61621638627012, "tpot_relative": 1.0, "ttft_ms": 578.1303280913562, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "d2213efae019eaec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 5722.431952087209, "interactivity_tok_s": 171.87838878284117, "output_per_gpu": 40.63945133197696, "total_per_gpu": 243.5425674218284, "tpot_ms": 5.818067105943404, "tpot_relative": 1.0, "ttft_ms": 315.64071119064465, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 7379.852157906862, "interactivity_tok_s": 132.39591648778125, "output_per_gpu": 123.62031632475484, "total_per_gpu": 742.5338650680218, "tpot_ms": 7.553103045230926, "tpot_relative": 1.298215, "ttft_ms": 430.7237211294705, "ttft_relative": 1.364601, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 8997.111242015671, "interactivity_tok_s": 109.0477433090212, "output_per_gpu": 201.1736830209112, "total_per_gpu": 1196.1931802436382, "tpot_ms": 9.170295227166548, "tpot_relative": 1.576176, "ttft_ms": 510.8833623235114, "ttft_relative": 1.61856, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 11659.32656958903, "interactivity_tok_s": 83.08366608498321, "output_per_gpu": 305.60895783959126, "total_per_gpu": 1840.2587341732378, "tpot_ms": 12.036060120134046, "tpot_relative": 2.068739, "ttft_ms": 651.2739193545713, "ttft_relative": 2.063339, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "39c72cbad7591f74", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28028985088", "measured": {"e2e_ms": 222533.7529318121, "interactivity_tok_s": 27.307913319749655, "output_per_gpu": 1410.9687141252987, "total_per_gpu": 6364.073433765502, "tpot_ms": 36.619421934255925, "tpot_relative": 1.0, "ttft_ms": 188832.35437804955, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "52fadace8cf8a372", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 147658.85198036424, "interactivity_tok_s": 42.07852819372644, "output_per_gpu": 1514.072813968693, "total_per_gpu": 9105.489465690172, "tpot_ms": 23.765089772058417, "tpot_relative": 1.0, "ttft_ms": 125785.15519790535, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b9a5ff547ff26238", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27951287552", "measured": {"e2e_ms": 18381.535861750854, "interactivity_tok_s": 62.94684829007009, "output_per_gpu": 383.36972639596553, "total_per_gpu": 3080.0193430654053, "tpot_ms": 15.88641889410928, "tpot_relative": 1.0, "ttft_ms": 3780.1500641282473, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 18}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "50d8773ce74b3b17", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27951287552", "measured": {"e2e_ms": 143366.16118046473, "interactivity_tok_s": 34.94040169538886, "output_per_gpu": 2076.227246702773, "total_per_gpu": 11237.609336838488, "tpot_ms": 28.62016323447053, "tpot_relative": 1.0, "ttft_ms": 117023.95529668554, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "fea430da80665c98", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 7123.700576572446, "interactivity_tok_s": 141.05140202316153, "output_per_gpu": 123.82380603905375, "total_per_gpu": 994.158644968103, "tpot_ms": 7.089614038971365, "tpot_relative": 1.0, "ttft_ms": 639.9183091692976, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 8953.077642912103, "interactivity_tok_s": 114.74345220904806, "output_per_gpu": 198.92203037434584, "total_per_gpu": 1587.6167316059064, "tpot_ms": 8.71509424501301, "tpot_relative": 1.229276, "ttft_ms": 906.5889408033399, "ttft_relative": 1.416726, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 11831.02072948168, "interactivity_tok_s": 90.60477031244699, "output_per_gpu": 300.64989233931385, "total_per_gpu": 2416.6614409265107, "tpot_ms": 11.036946471488635, "tpot_relative": 1.556777, "ttft_ms": 1672.560580253412, "ttft_relative": 2.61371, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 18722.072839468274, "interactivity_tok_s": 78.9492225078573, "output_per_gpu": 378.97109891852386, "total_per_gpu": 3044.680460569852, "tpot_ms": 12.666369195725474, "tpot_relative": 1.786609, "ttft_ms": 7080.371680517965, "ttft_relative": 11.064493, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 18}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f9cf1cf04f877053", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 49753.109869347376, "interactivity_tok_s": 50.856197783222676, "output_per_gpu": 1127.9027479273768, "total_per_gpu": 7396.861884371518, "tpot_ms": 19.663286749484392, "tpot_relative": 1.0, "ttft_ms": 31648.3187545786, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 97690.66310724556, "interactivity_tok_s": 50.80049025741115, "output_per_gpu": 1147.475913392921, "total_per_gpu": 7520.152985356305, "tpot_ms": 19.684849396785353, "tpot_relative": 1.001097, "ttft_ms": 79552.03254283017, "ttft_relative": 2.513626, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "311827b6ea0ec497", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 7052.843988827226, "interactivity_tok_s": 140.2493856330715, "output_per_gpu": 126.0061498693032, "total_per_gpu": 910.5122552523044, "tpot_ms": 7.130156010923693, "tpot_relative": 1.0, "ttft_ms": 531.4915077316982, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "315c69705b949786", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 5171.115071951499, "interactivity_tok_s": 200.128526143452, "output_per_gpu": 21.351923100064738, "total_per_gpu": 181.06134073388952, "tpot_ms": 4.996788909958796, "tpot_relative": 1.0, "ttft_ms": 572.6130572285911, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 34}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5d86883bbf8fc1a2", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27951287552", "measured": {"e2e_ms": 6994.216205202974, "interactivity_tok_s": 160.41154039625576, "output_per_gpu": 31.87484154026244, "total_per_gpu": 176.73084059801374, "tpot_ms": 6.233965446187695, "tpot_relative": 1.0, "ttft_ms": 1251.1740753485356, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27951287552", "measured": {"e2e_ms": 11574.607783504325, "interactivity_tok_s": 84.30933333920436, "output_per_gpu": 309.52360941860053, "total_per_gpu": 1722.4542614522038, "tpot_ms": 11.86108299512545, "tpot_relative": 1.902655, "ttft_ms": 658.5614065410482, "ttft_relative": 0.526355, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 26}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "13674befd22b0590", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208342156", "measured": {"e2e_ms": 8577.075304081882, "interactivity_tok_s": 115.36838816882681, "output_per_gpu": 206.7591690684014, "total_per_gpu": 1494.0283273302723, "tpot_ms": 8.667885682312113, "tpot_relative": 1.0, "ttft_ms": 649.814704104574, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "a172c3b8d95dafac", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4594.283486806671, "interactivity_tok_s": 209.46657916001018, "output_per_gpu": 50.61779231198998, "total_per_gpu": 453.10623159983635, "tpot_ms": 4.774031275109078, "tpot_relative": 1.0, "ttft_ms": 157.73028110561427, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5283.578181255143, "interactivity_tok_s": 182.78663146442392, "output_per_gpu": 88.24089721929687, "total_per_gpu": 786.3558896865745, "tpot_ms": 5.470859613683683, "tpot_relative": 1.145962, "ttft_ms": 171.60418002167717, "ttft_relative": 1.08796, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 6361.050203442574, "interactivity_tok_s": 148.82883141264378, "output_per_gpu": 141.4220012155079, "total_per_gpu": 1271.2994487748094, "tpot_ms": 6.719128212647142, "tpot_relative": 1.407433, "ttft_ms": 184.54302145401016, "ttft_relative": 1.169991, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 8024.798055175051, "interactivity_tok_s": 118.83449731822213, "output_per_gpu": 227.15972251300684, "total_per_gpu": 2020.690756590813, "tpot_ms": 8.415064838639744, "tpot_relative": 1.762675, "ttft_ms": 232.52502392533643, "ttft_relative": 1.474194, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 10862.212737931259, "interactivity_tok_s": 86.47868385746055, "output_per_gpu": 328.67638490790904, "total_per_gpu": 2966.2221070550013, "tpot_ms": 11.563543238565714, "tpot_relative": 2.422176, "ttft_ms": 282.0535895316425, "ttft_relative": 1.788202, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 15827.879032938654, "interactivity_tok_s": 59.98988594382967, "output_per_gpu": 457.8755310646986, "total_per_gpu": 4094.468789548259, "tpot_ms": 16.66947660037777, "tpot_relative": 3.491698, "ttft_ms": 402.27359423734015, "ttft_relative": 2.550389, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 23869.756046914335, "interactivity_tok_s": 39.68090743031187, "output_per_gpu": 608.1385226674417, "total_per_gpu": 5481.6256219277375, "tpot_ms": 25.201036588092474, "tpot_relative": 5.278775, "ttft_ms": 635.4834581643445, "ttft_relative": 4.028925, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 35753.18248221631, "interactivity_tok_s": 26.527560323901252, "output_per_gpu": 811.9192852692219, "total_per_gpu": 7330.073759080303, "tpot_ms": 37.696644085999985, "tpot_relative": 7.896187, "ttft_ms": 1053.6148582889155, "ttft_relative": 6.679852, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "1a6985264a00c441", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4727.307004708564, "interactivity_tok_s": 203.25653398309237, "output_per_gpu": 24.597326652113225, "total_per_gpu": 220.18348643248208, "tpot_ms": 4.919891038204871, "tpot_relative": 1.0, "ttft_ms": 155.2474769006949, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5382.482103200164, "interactivity_tok_s": 179.10976191997617, "output_per_gpu": 43.31494924124345, "total_per_gpu": 385.99976338271205, "tpot_ms": 5.5831686072297195, "tpot_relative": 1.134815, "ttft_ms": 165.02377468859777, "ttft_relative": 1.062972, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 6204.712432686938, "interactivity_tok_s": 152.49417798388225, "output_per_gpu": 72.44396294515408, "total_per_gpu": 651.2280222855306, "tpot_ms": 6.557627400737188, "tpot_relative": 1.332881, "ttft_ms": 176.91455848398618, "ttft_relative": 1.139565, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 7539.726671826793, "interactivity_tok_s": 126.4026925579538, "output_per_gpu": 120.78314981498245, "total_per_gpu": 1074.4219603855324, "tpot_ms": 7.911223881101382, "tpot_relative": 1.608008, "ttft_ms": 215.26700458489358, "ttft_relative": 1.386605, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 9729.403399728471, "interactivity_tok_s": 96.66536127137528, "output_per_gpu": 182.94402285733597, "total_per_gpu": 1651.0240159330274, "tpot_ms": 10.344967285568112, "tpot_relative": 2.102682, "ttft_ms": 264.1167999434401, "ttft_relative": 1.701263, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 13648.201880592387, "interactivity_tok_s": 69.68551816543544, "output_per_gpu": 264.9467950865462, "total_per_gpu": 2369.238602574321, "tpot_ms": 14.35018388793452, "tpot_relative": 2.916769, "ttft_ms": 366.0708813884412, "ttft_relative": 2.357983, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 20318.287743975816, "interactivity_tok_s": 46.67202898395674, "output_per_gpu": 357.02711516872307, "total_per_gpu": 3218.1631475137515, "tpot_ms": 21.42610942292105, "tpot_relative": 4.354997, "ttft_ms": 561.4030559852836, "ttft_relative": 3.616182, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 30420.15765551987, "interactivity_tok_s": 31.252100845708974, "output_per_gpu": 476.8823186289326, "total_per_gpu": 4305.332603095172, "tpot_ms": 31.997848878607584, "tpot_relative": 6.503772, "ttft_ms": 957.8798915195875, "ttft_relative": 6.170019, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 52352.67147729974, "interactivity_tok_s": 18.22101427243247, "output_per_gpu": 557.4205529141948, "total_per_gpu": 5015.342573733487, "tpot_ms": 54.88168688353164, "tpot_relative": 11.155061, "ttft_ms": 1703.6478574984358, "ttft_relative": 10.973756, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 90575.78141460406, "interactivity_tok_s": 10.576709089779584, "output_per_gpu": 645.4400496648859, "total_per_gpu": 5809.007337611106, "tpot_ms": 94.5473673816285, "tpot_relative": 19.21737, "ttft_ms": 3320.4577953175544, "ttft_relative": 21.388159, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "523115639410f745", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 32457.06579058642, "interactivity_tok_s": 29.074745368065184, "output_per_gpu": 446.7097767298479, "total_per_gpu": 4026.542747115221, "tpot_ms": 34.39411032979741, "tpot_relative": 1.0, "ttft_ms": 740.2819772552903, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 48493.50601816032, "interactivity_tok_s": 19.411590233505546, "output_per_gpu": 598.5706983239671, "total_per_gpu": 5403.945254587744, "tpot_ms": 51.51561453599722, "tpot_relative": 1.497803, "ttft_ms": 1042.6429738760817, "ttft_relative": 1.40844, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "1a0dfb901982014d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 24954.23488590095, "interactivity_tok_s": 37.92384005907237, "output_per_gpu": 581.217283866981, "total_per_gpu": 5238.963552543687, "tpot_ms": 26.368637734004313, "tpot_relative": 1.0, "ttft_ms": 645.7934643400222, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 37239.146080272985, "interactivity_tok_s": 25.467632293443387, "output_per_gpu": 778.8476283040688, "total_per_gpu": 7031.500133243603, "tpot_ms": 39.2655268647588, "tpot_relative": 1.4891, "ttft_ms": 1089.3705018350374, "ttft_relative": 1.686871, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 62087.30106462281, "interactivity_tok_s": 15.377399358918222, "output_per_gpu": 939.6320350723344, "total_per_gpu": 8454.256888277198, "tpot_ms": 65.03050201528671, "tpot_relative": 2.466206, "ttft_ms": 2013.7131611380025, "ttft_relative": 3.1182, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "600ba03d3dc41921", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4306.0384063079255, "interactivity_tok_s": 223.3754428752545, "output_per_gpu": 27.003801630967256, "total_per_gpu": 241.7250977771073, "tpot_ms": 4.476767844881036, "tpot_relative": 1.0, "ttft_ms": 145.71802829450462, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 4669.637709343806, "interactivity_tok_s": 207.17038271524828, "output_per_gpu": 49.91918348078687, "total_per_gpu": 444.85318231643316, "tpot_ms": 4.826944792463316, "tpot_relative": 1.078221, "ttft_ms": 158.81690592505038, "ttft_relative": 1.089892, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 5327.2453621531895, "interactivity_tok_s": 178.20515070752816, "output_per_gpu": 84.36558817997116, "total_per_gpu": 758.3963232518504, "tpot_ms": 5.611510082787723, "tpot_relative": 1.253474, "ttft_ms": 169.36077080099494, "ttft_relative": 1.16225, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 6478.0210898810765, "interactivity_tok_s": 147.6794898613677, "output_per_gpu": 140.6602202760303, "total_per_gpu": 1251.2376920848167, "tpot_ms": 6.771421007336481, "tpot_relative": 1.512569, "ttft_ms": 207.154349173652, "ttft_relative": 1.421611, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 8657.710144507291, "interactivity_tok_s": 108.87735990448621, "output_per_gpu": 205.69738806542563, "total_per_gpu": 1856.3674418352016, "tpot_ms": 9.184645925261783, "tpot_relative": 2.051624, "ttft_ms": 252.83311552775555, "ttft_relative": 1.735085, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 12291.652605007403, "interactivity_tok_s": 77.50943408192661, "output_per_gpu": 294.45709203534165, "total_per_gpu": 2633.1290741751513, "tpot_ms": 12.901655286800457, "tpot_relative": 2.881913, "ttft_ms": 348.0819157761289, "ttft_relative": 2.388736, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 19112.981585742546, "interactivity_tok_s": 49.65544992348123, "output_per_gpu": 379.7462836452734, "total_per_gpu": 3422.948687958876, "tpot_ms": 20.138776338568967, "tpot_relative": 4.498508, "ttft_ms": 544.081802101573, "ttft_relative": 3.733799, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "bc842c1413990d2a", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 41726.480755059674, "interactivity_tok_s": 22.56686527706445, "output_per_gpu": 347.30362524992677, "total_per_gpu": 3135.4855538796646, "tpot_ms": 44.312756234528386, "tpot_relative": 1.0, "ttft_ms": 899.6775398234604, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 68484.00568185306, "interactivity_tok_s": 13.767648626711294, "output_per_gpu": 426.531078091409, "total_per_gpu": 3837.6759948813346, "tpot_ms": 72.63404428116002, "tpot_relative": 1.639123, "ttft_ms": 1364.215157297258, "ttft_relative": 1.516338, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=b300_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208026609", "measured": {"e2e_ms": 106621.1752941722, "interactivity_tok_s": 8.857520430142747, "output_per_gpu": 549.0282851420375, "total_per_gpu": 4941.294452679277, "tpot_ms": 112.89841303633145, "tpot_relative": 2.547763, "ttft_ms": 2247.221627440558, "ttft_relative": 2.497808, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 16, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 16, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp8"], "rows": 16, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "303db00ad5d547be", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 66421.05833504471, "interactivity_tok_s": 26.462689208649103, "output_per_gpu": 3402.511138676835, "total_per_gpu": 13641.584933160733, "tpot_ms": 37.78905432155242, "tpot_relative": 1.0, "ttft_ms": 31640.74345897468, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "75639fc279496677", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 105839.88783551803, "interactivity_tok_s": 39.67916723454615, "output_per_gpu": 2116.4054379204804, "total_per_gpu": 10909.594633375295, "tpot_ms": 25.20214182139798, "tpot_relative": 1.0, "ttft_ms": 82643.60431689706, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1aa61038bd18361e", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 106916.00793743294, "interactivity_tok_s": 36.72486801142147, "output_per_gpu": 2016.7657518520007, "total_per_gpu": 10395.97443332895, "tpot_ms": 27.229505622429983, "tpot_relative": 1.0, "ttft_ms": 81853.87052683231, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "a2222a79a785ad35", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 40826.34243537832, "interactivity_tok_s": 51.60489589328441, "output_per_gpu": 689.2996920199618, "total_per_gpu": 4972.041413830469, "tpot_ms": 19.37800634397045, "tpot_relative": 1.0, "ttft_ms": 22978.634827179485, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "052dc890a498abbb", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 5345.515840407461, "interactivity_tok_s": 186.79170245379638, "output_per_gpu": 42.57514951105846, "total_per_gpu": 306.8766487143949, "tpot_ms": 5.353556859664865, "tpot_relative": 1.0, "ttft_ms": 420.04648158326745, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 6992.254202067852, "interactivity_tok_s": 142.95692311152038, "output_per_gpu": 126.3963560625177, "total_per_gpu": 913.3318598618068, "tpot_ms": 6.995114180093973, "tpot_relative": 1.306629, "ttft_ms": 594.7113901842386, "ttft_relative": 1.415823, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 8647.667825003737, "interactivity_tok_s": 117.25325163790593, "output_per_gpu": 206.38086222544513, "total_per_gpu": 1482.4317769238073, "tpot_ms": 8.528548130060706, "tpot_relative": 1.593062, "ttft_ms": 772.7249003830366, "ttft_relative": 1.839618, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 10734.793565020664, "interactivity_tok_s": 96.25586433911246, "output_per_gpu": 331.4112528610598, "total_per_gpu": 2397.5325940815774, "tpot_ms": 10.388977407931929, "tpot_relative": 1.940575, "ttft_ms": 1174.3197263000184, "ttft_relative": 2.79569, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 14383.681033182074, "interactivity_tok_s": 74.1494656393491, "output_per_gpu": 494.7752687238069, "total_per_gpu": 3577.5533744392487, "tpot_ms": 13.486273857506092, "tpot_relative": 2.519124, "ttft_ms": 1989.4199141766876, "ttft_relative": 4.73619, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "d893673eeed788f6", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 21830.599474164228, "interactivity_tok_s": 49.22357569440475, "output_per_gpu": 1291.73887708593, "total_per_gpu": 7764.618770664941, "tpot_ms": 20.31546846999313, "tpot_relative": 1.0, "ttft_ms": 3119.295425774544, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 41263.36988891917, "interactivity_tok_s": 43.88007863440688, "output_per_gpu": 1365.545218219979, "total_per_gpu": 8209.059050863803, "tpot_ms": 22.78938486714307, "tpot_relative": 1.121775, "ttft_ms": 20279.601450525388, "ttft_relative": 6.50134, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b00180feca7763d0", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208358162", "measured": {"e2e_ms": 5918.93988691736, "interactivity_tok_s": 166.35944466640288, "output_per_gpu": 76.8428671876978, "total_per_gpu": 461.56192503095895, "tpot_ms": 6.011080416896553, "tpot_relative": 1.0, "ttft_ms": 389.0464707976207, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1eac5eeda7d06e97", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 65080.50567306855, "interactivity_tok_s": 30.909507095019435, "output_per_gpu": 3451.474624219356, "total_per_gpu": 13837.892753893288, "tpot_ms": 32.35250555519644, "tpot_relative": 1.0, "ttft_ms": 35303.379136457115, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "2dfd89c56df6ee21", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 21692.930128054013, "interactivity_tok_s": 49.44073505124376, "output_per_gpu": 1302.5169069756025, "total_per_gpu": 7829.405311255042, "tpot_ms": 20.226236502421163, "tpot_relative": 1.0, "ttft_ms": 3063.995708893617, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 40634.62280629287, "interactivity_tok_s": 46.20332202850543, "output_per_gpu": 1385.422989272896, "total_per_gpu": 8328.555493893096, "tpot_ms": 21.64346536344386, "tpot_relative": 1.070069, "ttft_ms": 20705.461134811798, "ttft_relative": 6.757666, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cd88f602504da68c", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb200: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28205303704", "measured": {"e2e_ms": 40235.873319470695, "interactivity_tok_s": 57.11730966040564, "output_per_gpu": 698.9925666827922, "total_per_gpu": 5041.957844666862, "tpot_ms": 17.507827415989293, "tpot_relative": 1.0, "ttft_ms": 24110.27190327777, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 23, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 23, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 23, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "a498f62be1167e6d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 6205.5492575789685, "interactivity_tok_s": 157.83910278814128, "output_per_gpu": 36.45110032287735, "total_per_gpu": 202.10401964343708, "tpot_ms": 6.335565663612805, "tpot_relative": 1.0, "ttft_ms": 375.62191965116654, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 11317.465737056864, "interactivity_tok_s": 85.28414803565056, "output_per_gpu": 315.5523196412063, "total_per_gpu": 1756.0031646634743, "tpot_ms": 11.725508468255777, "tpot_relative": 1.850744, "ttft_ms": 525.6661889199677, "ttft_relative": 1.399456, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 26}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1777d8a5aeed4d25", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 34665.126344175216, "interactivity_tok_s": 41.627501233923134, "output_per_gpu": 1630.8790144738375, "total_per_gpu": 9803.183935312269, "tpot_ms": 24.022580514274992, "tpot_relative": 1.0, "ttft_ms": 12539.110561477477, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "4b6cfc0ad4d64618", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 18696.620434704662, "interactivity_tok_s": 55.20595687258994, "output_per_gpu": 753.4257691098945, "total_per_gpu": 5429.180612987073, "tpot_ms": 18.11398726966918, "tpot_relative": 1.0, "ttft_ms": 2005.6765098614048, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 31862.703441221176, "interactivity_tok_s": 51.88772989456167, "output_per_gpu": 883.9723173238484, "total_per_gpu": 6376.249723156095, "tpot_ms": 19.27237907752078, "tpot_relative": 1.06395, "ttft_ms": 14112.32299998128, "ttft_relative": 7.036191, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "82f011faa375b0c3", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28202535361", "measured": {"e2e_ms": 22102.703535130353, "interactivity_tok_s": 49.7074166856193, "output_per_gpu": 1271.0062045544578, "total_per_gpu": 8334.540060192036, "tpot_ms": 20.11772219676238, "tpot_relative": 1.0, "ttft_ms": 3573.42011456019, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ec9e286bd91e7b3e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 18055.14556224673, "interactivity_tok_s": 60.198510680525445, "output_per_gpu": 788.513629710123, "total_per_gpu": 5701.476559321521, "tpot_ms": 16.6117066468142, "tpot_relative": 1.0, "ttft_ms": 2787.592844330595, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cc6209bca81c2621", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 6723.884951096261, "interactivity_tok_s": 144.8210375558158, "output_per_gpu": 34.58735702771731, "total_per_gpu": 207.2738054964268, "tpot_ms": 6.905074130646162, "tpot_relative": 1.0, "ttft_ms": 306.95862849242985, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 8154.231112898561, "interactivity_tok_s": 118.31633628052685, "output_per_gpu": 111.77190898433695, "total_per_gpu": 671.3655979179154, "tpot_ms": 8.451918234089078, "tpot_relative": 1.224016, "ttft_ms": 378.1003078693175, "ttft_relative": 1.231763, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 9729.283235062758, "interactivity_tok_s": 99.59211215911029, "output_per_gpu": 185.90967563999556, "total_per_gpu": 1105.4322951315337, "tpot_ms": 10.040955837972195, "tpot_relative": 1.454142, "ttft_ms": 438.137897462002, "ttft_relative": 1.427352, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 12251.715427132513, "interactivity_tok_s": 78.39456443795775, "output_per_gpu": 290.5022373234695, "total_per_gpu": 1749.2919164103275, "tpot_ms": 12.755986428005606, "tpot_relative": 1.847335, "ttft_ms": 585.9516402437293, "ttft_relative": 1.908895, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "af5fa39a0046e159", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 17209.605360916023, "interactivity_tok_s": 62.08228082311455, "output_per_gpu": 409.2976355418914, "total_per_gpu": 3288.3259885729635, "tpot_ms": 16.107655626397005, "tpot_relative": 1.0, "ttft_ms": 2404.742119487355, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 18}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "4aa21b8da085ee48", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 123924.03206813075, "interactivity_tok_s": 41.05901604782834, "output_per_gpu": 1800.19374386324, "total_per_gpu": 10826.1934430896, "tpot_ms": 24.35518666192906, "tpot_relative": 1.0, "ttft_ms": 101507.32556010544, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "12de0f5660d68d82", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 44906.800918660774, "interactivity_tok_s": 37.99917257664337, "output_per_gpu": 2498.06929707875, "total_per_gpu": 12871.969022809844, "tpot_ms": 26.31636249402603, "tpot_relative": 1.0, "ttft_ms": 20676.095240017432, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "c98341fee35a3069", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 47603.508483097576, "interactivity_tok_s": 22.104635120377132, "output_per_gpu": 4800.4992022299375, "total_per_gpu": 17303.429546558713, "tpot_ms": 45.239380544135344, "tpot_relative": 1.0, "ttft_ms": 5918.253798430461, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e7c453296b2e3089", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 23179.49586903561, "interactivity_tok_s": 50.52438206868341, "output_per_gpu": 1212.2127095719723, "total_per_gpu": 7949.005562048499, "tpot_ms": 19.792424153561914, "tpot_relative": 1.0, "ttft_ms": 4950.0466076967, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "53a7f711391460da", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 6846.913745762959, "interactivity_tok_s": 143.0261683877179, "output_per_gpu": 130.018822154557, "total_per_gpu": 939.5075645750989, "tpot_ms": 6.99172753680419, "tpot_relative": 1.0, "ttft_ms": 452.6936698319332, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ba1a21295858ce17", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 8790.894391758957, "interactivity_tok_s": 116.84855985213702, "output_per_gpu": 202.41033217874707, "total_per_gpu": 1615.4572191533966, "tpot_ms": 8.558085792973607, "tpot_relative": 1.0, "ttft_ms": 889.197265524399, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 11169.231650232836, "interactivity_tok_s": 96.09156671873853, "output_per_gpu": 317.40929473933363, "total_per_gpu": 2551.3756137404785, "tpot_ms": 10.406740509570575, "tpot_relative": 1.216013, "ttft_ms": 1592.501103942186, "ttft_relative": 1.790942, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 17549.474131775547, "interactivity_tok_s": 77.95563920771863, "output_per_gpu": 404.99730579852564, "total_per_gpu": 3253.77683698595, "tpot_ms": 12.827808355665267, "tpot_relative": 1.498911, "ttft_ms": 5760.084976058533, "ttft_relative": 6.477848, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 18}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "527ce4c81d1e9712", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 5249.367161823466, "interactivity_tok_s": 186.9992254052957, "output_per_gpu": 42.76421399732358, "total_per_gpu": 308.2394031991649, "tpot_ms": 5.347615733876086, "tpot_relative": 1.0, "ttft_ms": 322.9222834477696, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ef0355cc906841fd", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30208366514", "measured": {"e2e_ms": 33807.0640062641, "interactivity_tok_s": 53.281359748245656, "output_per_gpu": 835.1437267647152, "total_per_gpu": 6018.039621561465, "tpot_ms": 18.768289786991144, "tpot_relative": 1.0, "ttft_ms": 16512.95293920578, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "4b61fb59f8967151", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=gb300: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28009839213", "measured": {"e2e_ms": 9380.126755917627, "interactivity_tok_s": 103.98616759023078, "output_per_gpu": 192.29348611240937, "total_per_gpu": 1255.6748071353197, "tpot_ms": 9.616663669543172, "tpot_relative": 1.0, "ttft_ms": 500.325634190267, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 16, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 16, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "h100", "precisions": ["fp8"], "rows": 16, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "65dac5f376ddc664", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8871.414972748607, "interactivity_tok_s": 111.80417997931445, "output_per_gpu": 13.106375588487968, "total_per_gpu": 117.32199650724755, "tpot_ms": 8.944209422089727, "tpot_relative": 1.0, "ttft_ms": 559.5063799060881, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10206.332326121628, "interactivity_tok_s": 97.25505744945649, "output_per_gpu": 22.842975098958544, "total_per_gpu": 203.5644307014328, "tpot_ms": 10.282241625528838, "tpot_relative": 1.149598, "ttft_ms": 595.7072309218347, "ttft_relative": 1.064701, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 12514.521807222627, "interactivity_tok_s": 77.53907814357305, "output_per_gpu": 35.99627873293438, "total_per_gpu": 323.5850781194132, "tpot_ms": 12.896722838881036, "tpot_relative": 1.441908, "ttft_ms": 663.5058334330097, "ttft_relative": 1.185877, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 16642.766973969992, "interactivity_tok_s": 58.58343906157365, "output_per_gpu": 54.87383701211667, "total_per_gpu": 488.1281505470494, "tpot_ms": 17.0696704737487, "tpot_relative": 1.908461, "ttft_ms": 825.0514477374963, "ttft_relative": 1.474606, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 23603.803720808355, "interactivity_tok_s": 40.70578360912019, "output_per_gpu": 75.87805935993939, "total_per_gpu": 684.7804936668814, "tpot_ms": 24.566533581629628, "tpot_relative": 2.746641, "ttft_ms": 1111.5343858546112, "ttft_relative": 1.986634, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 35417.50378708821, "interactivity_tok_s": 27.394713849831508, "output_per_gpu": 102.73329033311155, "total_per_gpu": 918.6737931560012, "tpot_ms": 36.50339278890298, "tpot_relative": 4.081232, "ttft_ms": 1595.0871224486036, "ttft_relative": 2.850883, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 69879.11442662007, "interactivity_tok_s": 21.52304519919753, "output_per_gpu": 102.56831607753615, "total_per_gpu": 924.5280284867997, "tpot_ms": 46.461826881136886, "tpot_relative": 5.194626, "ttft_ms": 26982.76381888427, "ttft_relative": 48.226016, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 137805.4987205025, "interactivity_tok_s": 21.71297895141113, "output_per_gpu": 103.29648945530464, "total_per_gpu": 932.5691611209552, "tpot_ms": 46.05540318708824, "tpot_relative": 5.149187, "ttft_ms": 95406.91028824949, "ttft_relative": 170.51979, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 279348.18408848514, "interactivity_tok_s": 21.957593930133115, "output_per_gpu": 101.19003852737868, "total_per_gpu": 910.4485035775409, "tpot_ms": 45.542330511343856, "tpot_relative": 5.091823, "ttft_ms": 237304.47919728505, "ttft_relative": 424.131856, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "d73b713311fdb86e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9284.373188111931, "interactivity_tok_s": 113.1860198428052, "output_per_gpu": 12.523529204087867, "total_per_gpu": 112.10463484893273, "tpot_ms": 8.83501338229596, "tpot_relative": 1.0, "ttft_ms": 1071.7697580344975, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7836.639161664061, "interactivity_tok_s": 127.09104041612144, "output_per_gpu": 29.748875679094546, "total_per_gpu": 265.10613943184126, "tpot_ms": 7.868375274337202, "tpot_relative": 0.89059, "ttft_ms": 482.45019044261426, "ttft_relative": 0.450144, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9781.156776449643, "interactivity_tok_s": 99.6969849667711, "output_per_gpu": 46.08088085832951, "total_per_gpu": 414.2396369075563, "tpot_ms": 10.030393600501549, "tpot_relative": 1.1353, "ttft_ms": 563.6893302667886, "ttft_relative": 0.525943, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 13397.361427167198, "interactivity_tok_s": 73.01457809798282, "output_per_gpu": 68.18943675228473, "total_per_gpu": 606.5765665591795, "tpot_ms": 13.695895067119851, "tpot_relative": 1.550184, "ttft_ms": 704.9192826380022, "ttft_relative": 0.657715, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 19141.049972124165, "interactivity_tok_s": 50.2354104166778, "output_per_gpu": 93.58337013892489, "total_per_gpu": 844.5664918596657, "tpot_ms": 19.90627710026645, "tpot_relative": 2.253112, "ttft_ms": 912.7240570960566, "ttft_relative": 0.851605, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 29744.53742527112, "interactivity_tok_s": 32.39641672574366, "output_per_gpu": 122.32892310169775, "total_per_gpu": 1093.904180759064, "tpot_ms": 30.86761133077272, "tpot_relative": 3.493782, "ttft_ms": 1154.5833239040803, "ttft_relative": 1.077268, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h100_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 58886.87661504264, "interactivity_tok_s": 25.428056535260605, "output_per_gpu": 121.7523987701758, "total_per_gpu": 1097.4490905499247, "tpot_ms": 39.326639006536695, "tpot_relative": 4.451226, "ttft_ms": 22595.03290562243, "ttft_relative": 21.081984, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 34, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 34, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "h200", "precisions": ["fp8"], "rows": 34, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "d0850681dcef564b", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 6787.009432911873, "interactivity_tok_s": 146.3431460124876, "output_per_gpu": 17.132182387039656, "total_per_gpu": 153.35909066571153, "tpot_ms": 6.833254766264687, "tpot_relative": 1.0, "ttft_ms": 436.93513944745064, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7624.587487149984, "interactivity_tok_s": 131.48215409049521, "output_per_gpu": 30.57205242611793, "total_per_gpu": 272.44185227784936, "tpot_ms": 7.6055948954998875, "tpot_relative": 1.113027, "ttft_ms": 513.1424044258893, "ttft_relative": 1.174413, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9168.554219743237, "interactivity_tok_s": 106.60879433304018, "output_per_gpu": 49.125024696861125, "total_per_gpu": 441.60467453009073, "tpot_ms": 9.380089196732245, "tpot_relative": 1.372712, "ttft_ms": 542.2166362870485, "ttft_relative": 1.240955, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 12226.5522853937, "interactivity_tok_s": 80.12425494188763, "output_per_gpu": 74.72707116362865, "total_per_gpu": 664.7318472525526, "tpot_ms": 12.480615273430976, "tpot_relative": 1.826453, "ttft_ms": 659.6133541548625, "ttft_relative": 1.509637, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 17370.522207044996, "interactivity_tok_s": 55.409407888000004, "output_per_gpu": 103.08388900442114, "total_per_gpu": 930.3062966686542, "tpot_ms": 18.047476739352952, "tpot_relative": 2.641125, "ttft_ms": 845.877461443888, "ttft_relative": 1.935934, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 26538.905576092657, "interactivity_tok_s": 36.58718838100694, "output_per_gpu": 137.16662766399043, "total_per_gpu": 1226.5876593838711, "tpot_ms": 27.3319717707283, "tpot_relative": 3.999847, "ttft_ms": 1215.1433692255523, "ttft_relative": 2.781061, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 43856.672214122955, "interactivity_tok_s": 21.989400387246135, "output_per_gpu": 166.24716845937368, "total_per_gpu": 1498.5150656180074, "tpot_ms": 45.47645603742795, "tpot_relative": 6.655168, "ttft_ms": 1890.9277117811143, "ttft_relative": 4.327708, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 72553.87834443536, "interactivity_tok_s": 13.267283943619683, "output_per_gpu": 201.03671341882307, "total_per_gpu": 1814.9759026285856, "tpot_ms": 75.37337741843582, "tpot_relative": 11.030377, "ttft_ms": 3104.150780296186, "ttft_relative": 7.104374, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "e1ba76b864137722", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 7294.674673303962, "interactivity_tok_s": 138.64053225910774, "output_per_gpu": 31.880781198581968, "total_per_gpu": 285.38148286500183, "tpot_ms": 7.212897871245058, "tpot_relative": 1.0, "ttft_ms": 591.5094977244735, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8598.339233268052, "interactivity_tok_s": 117.28858480744621, "output_per_gpu": 54.24233582013411, "total_per_gpu": 483.3788139814132, "tpot_ms": 8.525978906145978, "tpot_relative": 1.182046, "ttft_ms": 628.3613207750022, "ttft_relative": 1.062301, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10970.152836386114, "interactivity_tok_s": 89.5327990124754, "output_per_gpu": 82.24993444087283, "total_per_gpu": 739.3778578843902, "tpot_ms": 11.169091227234627, "tpot_relative": 1.548489, "ttft_ms": 705.9416698291898, "ttft_relative": 1.193458, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 15375.331262906548, "interactivity_tok_s": 64.10314387253938, "output_per_gpu": 118.95521345342755, "total_per_gpu": 1058.1616213229208, "tpot_ms": 15.599858908454907, "tpot_relative": 2.162773, "ttft_ms": 915.2918468578719, "ttft_relative": 1.547383, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 23205.087416351307, "interactivity_tok_s": 41.58854552556357, "output_per_gpu": 154.84835365805375, "total_per_gpu": 1397.4676336734215, "tpot_ms": 24.045082302417185, "tpot_relative": 3.333623, "ttft_ms": 1183.664776373189, "ttft_relative": 2.001092, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 36203.86032430979, "interactivity_tok_s": 26.75988286955657, "output_per_gpu": 201.43276540675464, "total_per_gpu": 1801.2759258668775, "tpot_ms": 37.369371341219576, "tpot_relative": 5.18091, "ttft_ms": 1589.6966506013996, "ttft_relative": 2.687525, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "fcf357fd2c086e32", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 285728.6811466023, "interactivity_tok_s": 3.3300363158673325, "output_per_gpu": 102.86735291852821, "total_per_gpu": 925.5399928157626, "tpot_ms": 300.29702536128127, "tpot_relative": 1.0, "ttft_ms": 7991.80982694088, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 569511.9210794513, "interactivity_tok_s": 2.47885588431087, "output_per_gpu": 102.26394790212484, "total_per_gpu": 920.3829605010665, "tpot_ms": 403.4119152828456, "tpot_relative": 1.343376, "ttft_ms": 196469.42139649836, "ttft_relative": 24.583846, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "783d60ab5097821b", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8232.27494545281, "interactivity_tok_s": 120.6347104714024, "output_per_gpu": 14.124446698603657, "total_per_gpu": 126.43528144393402, "tpot_ms": 8.289488125700434, "tpot_relative": 1.0, "ttft_ms": 528.6516660824418, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 9252.102332096547, "interactivity_tok_s": 107.43528469489078, "output_per_gpu": 25.19805640883106, "total_per_gpu": 224.5516613061556, "tpot_ms": 9.307928981059947, "tpot_relative": 1.122859, "ttft_ms": 552.2773479111493, "ttft_relative": 1.04469, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 11181.049027317204, "interactivity_tok_s": 87.57888404418098, "output_per_gpu": 40.28935193108626, "total_per_gpu": 362.1772458404984, "tpot_ms": 11.418277486791556, "tpot_relative": 1.377441, "ttft_ms": 681.2726512784138, "ttft_relative": 1.288699, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 14674.939876468852, "interactivity_tok_s": 66.78116530373639, "output_per_gpu": 62.23644134579144, "total_per_gpu": 553.621920115465, "tpot_ms": 14.974281976838315, "tpot_relative": 1.806418, "ttft_ms": 797.2475492395461, "ttft_relative": 1.508077, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 20832.42250106996, "interactivity_tok_s": 46.31000457205213, "output_per_gpu": 86.00598259474745, "total_per_gpu": 776.1824658714283, "tpot_ms": 21.59360616007141, "tpot_relative": 2.604938, "ttft_ms": 1056.1688020126894, "ttft_relative": 1.997854, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 31291.303783873445, "interactivity_tok_s": 31.03881647093598, "output_per_gpu": 116.32749598337968, "total_per_gpu": 1040.2375085707445, "tpot_ms": 32.21772328002186, "tpot_relative": 3.886576, "ttft_ms": 1437.515805743169, "ttft_relative": 2.719212, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 53709.346358476614, "interactivity_tok_s": 17.987225447441972, "output_per_gpu": 135.8841192290174, "total_per_gpu": 1224.8292811836757, "tpot_ms": 55.59501118846618, "tpot_relative": 6.706688, "ttft_ms": 2401.7550073214807, "ttft_relative": 4.543171, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 91107.4742531302, "interactivity_tok_s": 10.588614883945505, "output_per_gpu": 160.22688750421352, "total_per_gpu": 1446.5414541844596, "tpot_ms": 94.44105871828462, "tpot_relative": 11.39287, "ttft_ms": 4076.601537609531, "ttft_relative": 7.711319, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 181560.31995604426, "interactivity_tok_s": 6.62633620763038, "output_per_gpu": 160.6856860232687, "total_per_gpu": 1445.7553778540432, "tpot_ms": 150.91295833261174, "tpot_relative": 18.205341, "ttft_ms": 41966.98543149905, "ttft_relative": 79.384949, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "7ce3d393e8ad597e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 8599.657622724771, "interactivity_tok_s": 117.45849406248651, "output_per_gpu": 27.04319047199616, "total_per_gpu": 242.07768781532323, "tpot_ms": 8.513645675280086, "tpot_relative": 1.0, "ttft_ms": 687.8865208476782, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 10193.60172841698, "interactivity_tok_s": 99.5213592742005, "output_per_gpu": 45.75153150372154, "total_per_gpu": 407.71328707959583, "tpot_ms": 10.048094271349404, "tpot_relative": 1.180234, "ttft_ms": 797.204437572509, "ttft_relative": 1.158919, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 12923.086824035272, "interactivity_tok_s": 76.16470731387182, "output_per_gpu": 69.83020735449239, "total_per_gpu": 627.731918333666, "tpot_ms": 13.12944059351582, "tpot_relative": 1.542164, "ttft_ms": 851.5373050235212, "ttft_relative": 1.237904, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 17959.236578573473, "interactivity_tok_s": 54.8542743188261, "output_per_gpu": 101.8238285632343, "total_per_gpu": 905.770032214377, "tpot_ms": 18.230119939018095, "tpot_relative": 2.141282, "ttft_ms": 1053.836718504317, "ttft_relative": 1.531992, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 26410.66158026224, "interactivity_tok_s": 36.567111170173966, "output_per_gpu": 136.01608938173052, "total_per_gpu": 1227.5111622403322, "tpot_ms": 27.3469784185646, "tpot_relative": 3.212135, "ttft_ms": 1363.4563277009875, "ttft_relative": 1.982095, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 40969.80147198192, "interactivity_tok_s": 23.74406199352917, "output_per_gpu": 177.98624984373785, "total_per_gpu": 1591.6097181680332, "tpot_ms": 42.11579300426878, "tpot_relative": 4.946858, "ttft_ms": 1939.8502826457843, "ttft_relative": 2.820015, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 78045.04215041816, "interactivity_tok_s": 15.36886536419905, "output_per_gpu": 185.88878369259407, "total_per_gpu": 1675.5602244186855, "tpot_ms": 65.06661203041355, "tpot_relative": 7.642626, "ttft_ms": 18024.94370188797, "ttft_relative": 26.203368, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 155715.40883925263, "interactivity_tok_s": 15.172080118958506, "output_per_gpu": 183.78891803454593, "total_per_gpu": 1659.261394250632, "tpot_ms": 65.91054042421214, "tpot_relative": 7.741753, "ttft_ms": 95016.60380682697, "ttft_relative": 138.128312, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=vllm, gpu_sku=h200_sxm: Unsupported quant algorithm: mxfp8", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27513434236", "measured": {"e2e_ms": 305798.0862415694, "interactivity_tok_s": 15.642782113210444, "output_per_gpu": 185.59892284601216, "total_per_gpu": 1669.9100428253648, "tpot_ms": 63.92724726092634, "tpot_relative": 7.508798, "ttft_ms": 246788.15544812896, "ttft_relative": 358.762889, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 174}]}, {"aic": {"coverage_pct": 100.0, "points": 25, "status_counts": {"failed": 0, "success": 25, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 50.6, "tpot_shape_error_pct": 15.59, "ttft_mape_pct": 63.96, "ttft_shape_error_pct": 43.31}, "aisimulate": {"coverage_pct": 100.0, "points": 25, "status_counts": {"failed": 0, "success": 25, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 47.49, "tpot_shape_error_pct": 16.8, "ttft_mape_pct": 42.31, "ttft_shape_error_pct": 25.36}, "frameworks": ["vllm"], "gpu_skus": ["b200", "b300", "h200"], "hf_model_paths": ["moonshotai/Kimi-K2.5"], "model": "moonshotai/Kimi-K2.5", "precisions": ["int4"], "rows": 25, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.02, "tpot_shape_error_pct": 5.71, "ttft_mape_pct": 21.39, "ttft_shape_error_pct": 18.53}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.33, "tpot_shape_error_pct": 4.29, "ttft_mape_pct": 11.08, "ttft_shape_error_pct": 9.12}, "gpu_skus": ["h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.02, "tpot_shape_error_pct": 5.71, "ttft_mape_pct": 21.39, "ttft_shape_error_pct": 18.53}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.33, "tpot_shape_error_pct": 4.29, "ttft_mape_pct": 11.08, "ttft_shape_error_pct": 9.12}, "gpu": "h200", "precisions": ["int4"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.02, "tpot_shape_error_pct": 5.71, "ttft_mape_pct": 21.39, "ttft_shape_error_pct": 18.53}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.33, "tpot_shape_error_pct": 4.29, "ttft_mape_pct": 11.08, "ttft_shape_error_pct": 9.12}, "framework": "vllm", "id": "788ee91b1053a8c7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 13422.836795359866, "interactivity_tok_s": 77.27179369192388, "output_per_gpu": 38.106698889225875, "total_per_gpu": null, "tpot_error_pct": 37.28, "tpot_ms": 12.941332823033923, "tpot_relative": 1.372846, "ttft_error_pct": 26.66, "ttft_ms": 183.85331739616433, "ttft_relative": 1.266605, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11853.585280149946, "interactivity_tok_s": 78.13764775263344, "output_per_gpu": 37.62600279227994, "total_per_gpu": 75.6199735216442, "tpot_error_pct": 35.76, "tpot_ms": 12.79792812762651, "tpot_relative": 1.357633, "ttft_error_pct": 17.12, "ttft_ms": 120.3004927249996, "ttft_relative": 0.828776, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 8788.582797441632, "interactivity_tok_s": 106.08228234325128, "output_per_gpu": 50.7232635684782, "total_per_gpu": 101.9425812822909, "tpot_ms": 9.426644845029749, "tpot_relative": 1.0, "ttft_ms": 145.15441292896867, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16507.59599988419, "interactivity_tok_s": 62.74139084553106, "output_per_gpu": 61.97147058888386, "total_per_gpu": null, "tpot_error_pct": 37.32, "tpot_ms": 15.938441697315799, "tpot_relative": 1.690786, "ttft_error_pct": 20.77, "ttft_ms": 202.5701435301263, "ttft_relative": 1.395549, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14831.258260200004, "interactivity_tok_s": 63.20506442675774, "output_per_gpu": 61.481994482947904, "total_per_gpu": 122.50985965756307, "tpot_error_pct": 36.31, "tpot_ms": 15.82151697920985, "tpot_relative": 1.678383, "ttft_error_pct": 15.62, "ttft_ms": 141.53597321249933, "ttft_relative": 0.975072, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 10943.786948826164, "interactivity_tok_s": 86.15609792440856, "output_per_gpu": 83.27613911832931, "total_per_gpu": 165.9371691176967, "tpot_ms": 11.606839493559443, "tpot_relative": 1.23128, "ttft_ms": 167.73361262166873, "ttft_relative": 1.155553, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21557.69259343835, "interactivity_tok_s": 47.94716846167708, "output_per_gpu": 94.9081164940052, "total_per_gpu": null, "tpot_error_pct": 34.84, "tpot_ms": 20.85628895477475, "tpot_relative": 2.212483, "ttft_error_pct": 25.74, "ttft_ms": 221.70899270378032, "ttft_relative": 1.527401, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19239.04102513746, "interactivity_tok_s": 48.00127405286674, "output_per_gpu": 93.43163888866927, "total_per_gpu": 187.85788930205365, "tpot_error_pct": 34.68, "tpot_ms": 20.83278037367589, "tpot_relative": 2.209989, "ttft_error_pct": 2.41, "ttft_ms": 172.08399403749968, "ttft_relative": 1.185524, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 14333.988135671825, "interactivity_tok_s": 64.65009593559797, "output_per_gpu": 124.87259088574987, "total_per_gpu": 251.07449290735593, "tpot_ms": 15.46788114585573, "tpot_relative": 1.640868, "ttft_ms": 176.33013413433218, "ttft_relative": 1.214776, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27419.534972277674, "interactivity_tok_s": 37.64104023487004, "output_per_gpu": 149.23666663702312, "total_per_gpu": null, "tpot_error_pct": 21.94, "tpot_ms": 26.566747192964566, "tpot_relative": 2.818261, "ttft_error_pct": 7.3, "ttft_ms": 241.75259387491926, "ttft_relative": 1.665486, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24964.989650171705, "interactivity_tok_s": 37.310928081993694, "output_per_gpu": 144.9742066644972, "total_per_gpu": 289.48982200129973, "tpot_error_pct": 23.01, "tpot_ms": 26.80179913516012, "tpot_relative": 2.843196, "ttft_error_pct": 13.06, "ttft_ms": 226.7316228343745, "ttft_relative": 1.562003, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 20369.712521578185, "interactivity_tok_s": 45.897999072149396, "output_per_gpu": 177.23066231567904, "total_per_gpu": 353.9006976990953, "tpot_ms": 21.787442159037244, "tpot_relative": 2.311262, "ttft_ms": 260.78768034640234, "ttft_relative": 1.796622, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37773.9334830625, "interactivity_tok_s": 27.272577960688636, "output_per_gpu": 216.65734132956086, "total_per_gpu": null, "tpot_error_pct": 23.74, "tpot_ms": 36.66686740950652, "tpot_relative": 3.889705, "ttft_error_pct": 26.46, "ttft_ms": 263.72812313732686, "ttft_relative": 1.81688, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34972.325287578045, "interactivity_tok_s": 26.602782711501522, "output_per_gpu": 206.52670519072643, "total_per_gpu": 413.14652857669074, "tpot_error_pct": 26.85, "tpot_ms": 37.59005254618184, "tpot_relative": 3.987639, "ttft_error_pct": 7.17, "ttft_ms": 332.93237508281277, "ttft_relative": 2.293643, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 27663.34996256919, "interactivity_tok_s": 33.746425264875406, "output_per_gpu": 260.5762665290102, "total_per_gpu": 521.2700209714532, "tpot_ms": 29.63276827548425, "tpot_relative": 3.143512, "ttft_ms": 358.6359859822551, "ttft_relative": 2.470721, "unavailable_metrics": {}}, "status": "success"}], "precision": "int4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["int4"], "rows": 5}, {"aic": {"coverage_pct": 100.0, "points": 20, "status_counts": {"failed": 0, "success": 20, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 55.49, "tpot_shape_error_pct": 18.06, "ttft_mape_pct": 74.61, "ttft_shape_error_pct": 49.51}, "aisimulate": {"coverage_pct": 100.0, "points": 20, "status_counts": {"failed": 0, "success": 20, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.52, "tpot_shape_error_pct": 19.93, "ttft_mape_pct": 50.12, "ttft_shape_error_pct": 29.41}, "gpu_skus": ["b200", "b300", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 49.44, "tpot_shape_error_pct": 30.55, "ttft_mape_pct": 84.93, "ttft_shape_error_pct": 53.07}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.36, "tpot_shape_error_pct": 33.98, "ttft_mape_pct": 59.11, "ttft_shape_error_pct": 36.56}, "gpu": "b200", "precisions": ["int4"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 49.44, "tpot_shape_error_pct": 30.55, "ttft_mape_pct": 84.93, "ttft_shape_error_pct": 53.07}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.36, "tpot_shape_error_pct": 33.98, "ttft_mape_pct": 59.11, "ttft_shape_error_pct": 36.56}, "framework": "vllm", "id": "b01dd48b6e357b87", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12139.988124305659, "interactivity_tok_s": 88.3349823040167, "output_per_gpu": 42.13348437927364, "total_per_gpu": null, "tpot_error_pct": 20.09, "tpot_ms": 11.320543389688648, "tpot_relative": 1.20092, "ttft_error_pct": 143.09, "ttft_ms": 559.0722366541715, "ttft_relative": 2.430866, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10334.604340375017, "interactivity_tok_s": 92.8212720022547, "output_per_gpu": 43.60848806082668, "total_per_gpu": 392.01430015934824, "tpot_error_pct": 14.29, "tpot_ms": 10.773392547084566, "tpot_relative": 1.142876, "ttft_error_pct": 87.71, "ttft_ms": 431.70177594999984, "ttft_relative": 1.877054, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 8895.699126799855, "interactivity_tok_s": 106.08322082530395, "output_per_gpu": 50.50976065336404, "total_per_gpu": 454.0526249414145, "tpot_ms": 9.426561450719742, "tpot_relative": 1.0, "ttft_ms": 229.9889559497842, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16763.70149109019, "interactivity_tok_s": 63.34959791641644, "output_per_gpu": 61.024708686426955, "total_per_gpu": null, "tpot_error_pct": 31.66, "tpot_ms": 15.785419843065169, "tpot_relative": 1.674568, "ttft_error_pct": 117.8, "ttft_ms": 615.2169916345202, "ttft_relative": 2.674985, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14518.197002712506, "interactivity_tok_s": 66.24633257408688, "output_per_gpu": 62.95521094302247, "total_per_gpu": 560.0157079981741, "tpot_error_pct": 25.9, "tpot_ms": 15.09517525187746, "tpot_relative": 1.601345, "ttft_error_pct": 87.65, "ttft_ms": 530.0637147999996, "ttft_relative": 2.304736, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 11383.774153637478, "interactivity_tok_s": 83.40435917136529, "output_per_gpu": 80.0316720269421, "total_per_gpu": 711.9187244564873, "tpot_ms": 11.989780989089164, "tpot_relative": 1.271915, "ttft_ms": 282.47348844995486, "ttft_relative": 1.228205, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26924.143580209573, "interactivity_tok_s": 38.96774778562939, "output_per_gpu": 75.99127503925138, "total_per_gpu": null, "tpot_error_pct": 60.74, "tpot_ms": 25.6622478030096, "tpot_relative": 2.722334, "ttft_error_pct": 51.15, "ttft_ms": 671.6640777307549, "ttft_relative": 2.920419, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23521.706140218746, "interactivity_tok_s": 40.07681028839029, "output_per_gpu": 76.39592744452894, "total_per_gpu": 689.4541235094337, "tpot_error_pct": 56.29, "tpot_ms": 24.952085577770806, "tpot_relative": 2.646998, "ttft_error_pct": 54.42, "ttft_ms": 686.2019177624998, "ttft_relative": 2.98363, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 13975.881671607516, "interactivity_tok_s": 62.63510428464106, "output_per_gpu": 112.01333879722874, "total_per_gpu": 1145.2410511410467, "tpot_ms": 15.965487906838417, "tpot_relative": 1.69367, "ttft_ms": 444.3763660935058, "ttft_relative": 1.932164, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 38909.16439216997, "interactivity_tok_s": 26.79384861515426, "output_per_gpu": 105.16802568043504, "total_per_gpu": null, "tpot_error_pct": 66.61, "tpot_ms": 37.32199932765212, "tpot_relative": 3.959238, "ttft_error_pct": 32.92, "ttft_ms": 728.7590799818526, "ttft_relative": 3.16867, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34859.00617392493, "interactivity_tok_s": 27.345797318499407, "output_per_gpu": 104.48243756163089, "total_per_gpu": 934.3152246141179, "tpot_error_pct": 63.25, "tpot_ms": 36.568690550613454, "tpot_relative": 3.879324, "ttft_error_pct": 7.85, "ttft_ms": 1001.0889567218744, "ttft_relative": 4.35277, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 19552.65056889722, "interactivity_tok_s": 44.64215750018603, "output_per_gpu": 130.01486690485714, "total_per_gpu": 1594.3732099201043, "tpot_ms": 22.40035105820844, "tpot_relative": 2.376301, "ttft_ms": 1086.389137897224, "ttft_relative": 4.723658, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 58069.38902158352, "interactivity_tok_s": 17.858965774492837, "output_per_gpu": 140.93483912768792, "total_per_gpu": null, "tpot_error_pct": 68.11, "tpot_ms": 55.99428391470772, "tpot_relative": 5.940054, "ttft_error_pct": 79.69, "ttft_ms": 787.236576837514, "ttft_relative": 3.422932, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52958.67676283107, "interactivity_tok_s": 17.971286040151785, "output_per_gpu": 137.46408410325844, "total_per_gpu": 1239.0707337698357, "tpot_error_pct": 67.06, "tpot_ms": 55.644320488015225, "tpot_relative": 5.902929, "ttft_error_pct": 57.92, "ttft_ms": 1631.5674742984393, "ttft_relative": 7.094112, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 25328.115352627472, "interactivity_tok_s": 30.022472582207268, "output_per_gpu": 103.93138314848534, "total_per_gpu": 2299.342586164575, "tpot_ms": 33.30838248787833, "tpot_relative": 3.53346, "ttft_ms": 3876.870259754879, "ttft_relative": 16.856767, "unavailable_metrics": {}}, "status": "success"}], "precision": "int4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 69.81, "tpot_shape_error_pct": 17.07, "ttft_mape_pct": 93.04, "ttft_shape_error_pct": 51.9}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 65.84, "tpot_shape_error_pct": 19.75, "ttft_mape_pct": 68.17, "ttft_shape_error_pct": 36.13}, "gpu": "b300", "precisions": ["int4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 67.33, "tpot_shape_error_pct": 15.02, "ttft_mape_pct": 98.43, "ttft_shape_error_pct": 54.48}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 63.09, "tpot_shape_error_pct": 17.76, "ttft_mape_pct": 74.79, "ttft_shape_error_pct": 38.38}, "framework": "vllm", "id": "b76996cc062ca996", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 17418.81272833259, "interactivity_tok_s": 61.44013713888073, "output_per_gpu": 58.72960551071534, "total_per_gpu": null, "tpot_error_pct": 49.38, "tpot_ms": 16.276005337351, "tpot_relative": 1.493758, "ttft_error_pct": 178.41, "ttft_ms": 768.4592682225167, "ttft_relative": 2.784136, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14916.785213249977, "interactivity_tok_s": 64.26866288750574, "output_per_gpu": 60.42391720422596, "total_per_gpu": 543.1749796659199, "tpot_error_pct": 42.8, "tpot_ms": 15.559682667591437, "tpot_relative": 1.428017, "ttft_error_pct": 122.58, "ttft_ms": 614.3639496250007, "ttft_relative": 2.225847, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 10291.233457118506, "interactivity_tok_s": 91.77671320600093, "output_per_gpu": 87.50895520979307, "total_per_gpu": 786.6533181887268, "tpot_ms": 10.896010164968663, "tpot_relative": 1.0, "ttft_ms": 276.0135621210793, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24183.051164447756, "interactivity_tok_s": 43.835173071776005, "output_per_gpu": 84.60470873120788, "total_per_gpu": null, "tpot_error_pct": 65.68, "tpot_ms": 22.812730734804976, "tpot_relative": 2.093677, "ttft_error_pct": 136.14, "ttft_ms": 845.627622742265, "ttft_relative": 3.063718, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21073.060321362493, "interactivity_tok_s": 45.60586825929819, "output_per_gpu": 86.7475505688692, "total_per_gpu": 771.6595691006413, "tpot_error_pct": 59.24, "tpot_ms": 21.92700277767694, "tpot_relative": 2.012388, "ttft_error_pct": 110.77, "ttft_ms": 754.7887508124993, "ttft_relative": 2.734607, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 13109.576126416505, "interactivity_tok_s": 72.62494267436735, "output_per_gpu": 139.1601081436178, "total_per_gpu": 1237.893501106409, "tpot_ms": 13.769374035636185, "tpot_relative": 1.263708, "ttft_ms": 358.1117635156261, "ttft_relative": 1.297443, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35437.305294853984, "interactivity_tok_s": 29.6400610752939, "output_per_gpu": 115.4715339090475, "total_per_gpu": null, "tpot_error_pct": 67.46, "tpot_ms": 33.738122113167215, "tpot_relative": 3.096374, "ttft_error_pct": 74.19, "ttft_ms": 923.2063730839184, "ttft_relative": 3.344786, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31050.354991925047, "interactivity_tok_s": 30.43250241811068, "output_per_gpu": 115.57193570118231, "total_per_gpu": 1043.0077923853207, "tpot_error_pct": 63.1, "tpot_ms": 32.85960471672846, "tpot_relative": 3.015747, "ttft_error_pct": 84.13, "ttft_ms": 975.879729531251, "ttft_relative": 3.535622, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 17594.084351350746, "interactivity_tok_s": 49.63612868034889, "output_per_gpu": 177.86119614313, "total_per_gpu": 1835.1059185201952, "tpot_ms": 20.14661551145312, "tpot_relative": 1.84899, "ttft_ms": 529.9988107855876, "ttft_relative": 1.920191, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 52561.74476433877, "interactivity_tok_s": 19.84093115996511, "output_per_gpu": 155.70259390537862, "total_per_gpu": null, "tpot_error_pct": 72.66, "tpot_ms": 50.400860319388286, "tpot_relative": 4.625625, "ttft_error_pct": 26.06, "ttft_ms": 1001.6646576045508, "ttft_relative": 3.629041, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 47200.868451453105, "interactivity_tok_s": 20.22754559579142, "output_per_gpu": 154.29478535018342, "total_per_gpu": 1379.7530991388703, "tpot_error_pct": 69.36, "tpot_ms": 49.437535328461294, "tpot_relative": 4.537215, "ttft_error_pct": 5.12, "ttft_ms": 1423.9460089156241, "ttft_relative": 5.158971, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 25324.596699904956, "interactivity_tok_s": 34.25639784239397, "output_per_gpu": 203.6229602452194, "total_per_gpu": 2497.0297664280097, "tpot_ms": 29.191627345080953, "tpot_relative": 2.679112, "ttft_ms": 1354.6083307657477, "ttft_relative": 4.90776, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 82574.22487532343, "interactivity_tok_s": 12.553344797199664, "output_per_gpu": 198.22166087194395, "total_per_gpu": null, "tpot_error_pct": 81.46, "tpot_ms": 79.66004408825566, "tpot_relative": 7.310937, "ttft_error_pct": 77.36, "ttft_ms": 1081.999773037894, "ttft_relative": 3.920096, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 75600.22170330283, "interactivity_tok_s": 12.588013198216009, "output_per_gpu": 192.7349706996059, "total_per_gpu": 1737.270234081512, "tpot_error_pct": 80.96, "tpot_ms": 79.44065391842149, "tpot_relative": 7.290802, "ttft_error_pct": 51.37, "ttft_ms": 2324.302674548437, "ttft_relative": 8.420973, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 32719.66602468256, "interactivity_tok_s": 22.779831902797262, "output_per_gpu": 174.94782273108882, "total_per_gpu": 3677.4075484117366, "tpot_ms": 43.89848021122599, "tpot_relative": 4.028858, "ttft_ms": 4779.418711894074, "ttft_relative": 17.315884, "unavailable_metrics": {}}, "status": "success"}], "precision": "int4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.3, "tpot_shape_error_pct": 19.13, "ttft_mape_pct": 87.65, "ttft_shape_error_pct": 49.31}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 68.6, "tpot_shape_error_pct": 21.73, "ttft_mape_pct": 61.55, "ttft_shape_error_pct": 33.88}, "framework": "vllm", "id": "44fd3837fe41044b", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 14571.50242816649, "interactivity_tok_s": 72.94970876922001, "output_per_gpu": 35.102763254616654, "total_per_gpu": null, "tpot_error_pct": 49.43, "tpot_ms": 13.708073916560094, "tpot_relative": 1.494296, "ttft_error_pct": 139.73, "ttft_ms": 548.1428115255125, "ttft_relative": 2.397322, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12536.731533125, "interactivity_tok_s": 75.89785868596248, "output_per_gpu": 35.90579590524603, "total_per_gpu": 322.77169146121923, "tpot_error_pct": 43.63, "tpot_ms": 13.175602280660295, "tpot_relative": 1.436252, "ttft_error_pct": 86.3, "ttft_ms": 425.96201287499935, "ttft_relative": 1.86296, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 8659.987987345085, "interactivity_tok_s": 109.00843636393671, "output_per_gpu": 51.892989546759345, "total_per_gpu": 466.48702775419275, "tpot_ms": 9.173601909685132, "tpot_relative": 1.0, "ttft_ms": 228.64797431975603, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19456.389800855242, "interactivity_tok_s": 54.261334390362485, "output_per_gpu": 52.5791274985163, "total_per_gpu": null, "tpot_error_pct": 58.02, "tpot_ms": 18.42932930483945, "tpot_relative": 2.008952, "ttft_error_pct": 126.72, "ttft_ms": 603.1859220044872, "ttft_relative": 2.638055, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16972.937582362523, "interactivity_tok_s": 56.31843909540115, "output_per_gpu": 53.79737985804286, "total_per_gpu": 478.55256647325547, "tpot_error_pct": 52.25, "tpot_ms": 17.756173929217756, "tpot_relative": 1.935573, "ttft_error_pct": 96.89, "ttft_ms": 523.8117994249993, "ttft_relative": 2.290909, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 11063.604090724402, "interactivity_tok_s": 85.74634018926038, "output_per_gpu": 82.34756005337354, "total_per_gpu": 732.5196191773434, "tpot_ms": 11.662305327466894, "tpot_relative": 1.27129, "ttft_ms": 266.04686896607745, "ttft_relative": 1.163565, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29583.324842927832, "interactivity_tok_s": 35.36756828319793, "output_per_gpu": 69.16058322934298, "total_per_gpu": null, "tpot_error_pct": 82.07, "tpot_ms": 28.27449125121418, "tpot_relative": 3.082158, "ttft_error_pct": 65.58, "ttft_ms": 658.520292935727, "ttft_relative": 2.880062, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25956.459846256257, "interactivity_tok_s": 36.20092938452588, "output_per_gpu": 69.06415224193542, "total_per_gpu": 623.2866874279397, "tpot_error_pct": 77.88, "tpot_ms": 27.623600194846126, "tpot_relative": 3.011205, "ttft_error_pct": 70.81, "ttft_ms": 679.32125923125, "ttft_relative": 2.971036, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 13537.94674126749, "interactivity_tok_s": 64.39323337996737, "output_per_gpu": 115.33629653248688, "total_per_gpu": 1189.9971718206743, "tpot_ms": 15.529582030758814, "tpot_relative": 1.692855, "ttft_ms": 397.7044056622294, "ttft_relative": 1.739374, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 41784.631678282974, "interactivity_tok_s": 24.90859980116424, "output_per_gpu": 97.93074237212349, "total_per_gpu": null, "tpot_error_pct": 85.31, "tpot_ms": 40.146776935781816, "tpot_relative": 4.376337, "ttft_error_pct": 28.65, "ttft_ms": 714.4788729781826, "ttft_relative": 3.124799, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37604.539499615654, "interactivity_tok_s": 25.28505040383718, "output_per_gpu": 96.68508508838177, "total_per_gpu": 864.5888160668259, "tpot_error_pct": 82.55, "tpot_ms": 39.54906096798776, "tpot_relative": 4.311181, "ttft_error_pct": 0.86, "ttft_ms": 992.7867112437501, "ttft_relative": 4.341988, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 18800.926752819323, "interactivity_tok_s": 46.15829586361807, "output_per_gpu": 136.67138914670062, "total_per_gpu": 1676.0021880999534, "tpot_ms": 21.66457797650626, "tpot_relative": 2.361622, "ttft_ms": 1001.4306820138079, "ttft_relative": 4.379792, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 61904.14316995718, "interactivity_tok_s": 16.734177614719695, "output_per_gpu": 132.20439829901068, "total_per_gpu": null, "tpot_error_pct": 86.64, "tpot_ms": 59.75794108462081, "tpot_relative": 6.51412, "ttft_error_pct": 77.58, "ttft_ms": 771.7694403900869, "ttft_relative": 3.375361, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56742.91096517337, "interactivity_tok_s": 16.731426942896107, "output_per_gpu": 128.16287981463265, "total_per_gpu": 1155.2317434034965, "tpot_error_pct": 86.67, "tpot_ms": 59.76776538026147, "tpot_relative": 6.515191, "ttft_error_pct": 52.9, "ttft_ms": 1621.3799275078113, "ttft_relative": 7.091162, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29650514816", "measured": {"e2e_ms": 23974.492696638892, "interactivity_tok_s": 31.233255123822246, "output_per_gpu": 112.23959687527011, "total_per_gpu": 2483.7543769230083, "tpot_ms": 32.017155945980136, "tpot_relative": 3.49014, "ttft_ms": 3442.4229105843197, "ttft_relative": 15.055558, "unavailable_metrics": {}}, "status": "success"}], "precision": "int4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.89, "tpot_shape_error_pct": 7.52, "ttft_mape_pct": 27.42, "ttft_shape_error_pct": 41.17}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.05, "tpot_shape_error_pct": 6.24, "ttft_mape_pct": 5.04, "ttft_shape_error_pct": 8.84}, "gpu": "h200", "precisions": ["int4"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.89, "tpot_shape_error_pct": 7.52, "ttft_mape_pct": 27.42, "ttft_shape_error_pct": 41.17}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.05, "tpot_shape_error_pct": 6.24, "ttft_mape_pct": 5.04, "ttft_shape_error_pct": 8.84}, "framework": "vllm", "id": "33158ff68a624bf6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 16172.802434640607, "interactivity_tok_s": 66.05737528912775, "output_per_gpu": 31.627171732736656, "total_per_gpu": null, "tpot_error_pct": 41.04, "tpot_ms": 15.138355037920922, "tpot_relative": 1.410384, "ttft_error_pct": 31.12, "ttft_ms": 686.265230847504, "ttft_relative": 1.311184, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13903.228156300032, "interactivity_tok_s": 68.80326854141389, "output_per_gpu": 32.402480811962384, "total_per_gpu": 291.27897810193764, "tpot_error_pct": 35.41, "tpot_ms": 14.53419323237649, "tpot_relative": 1.354097, "ttft_error_pct": 3.85, "ttft_ms": 543.5394732500012, "ttft_relative": 1.038491, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 10389.95244242251, "interactivity_tok_s": 93.16629171666091, "output_per_gpu": 43.34207806609316, "total_per_gpu": 389.61943318998107, "tpot_ms": 10.73349579095859, "tpot_relative": 1.0, "ttft_ms": 523.3936227159575, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21218.841981612237, "interactivity_tok_s": 49.99107345019061, "output_per_gpu": 48.21186758855683, "total_per_gpu": null, "tpot_error_pct": 41.89, "tpot_ms": 20.003571257503914, "tpot_relative": 1.863659, "ttft_error_pct": 7.24, "ttft_ms": 755.1885851857348, "ttft_relative": 1.442869, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18490.400629425032, "interactivity_tok_s": 51.990463483629014, "output_per_gpu": 49.39275624370332, "total_per_gpu": 439.37140299367826, "tpot_error_pct": 36.43, "tpot_ms": 19.234296695871627, "tpot_relative": 1.791988, "ttft_error_pct": 5.13, "ttft_ms": 668.0641623125009, "ttft_relative": 1.276409, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 13769.776809716132, "interactivity_tok_s": 70.92998851807303, "output_per_gpu": 66.3172628119822, "total_per_gpu": 589.9227137808423, "tpot_ms": 14.09840916222902, "tpot_relative": 1.313497, "ttft_ms": 704.1841401252896, "ttft_relative": 1.34542, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30470.462065049156, "interactivity_tok_s": 34.507217030878586, "output_per_gpu": 67.1469961837843, "total_per_gpu": null, "tpot_error_pct": 37.91, "tpot_ms": 28.979445056527034, "tpot_relative": 2.699907, "ttft_error_pct": 9.55, "ttft_ms": 824.4897722219996, "ttft_relative": 1.575277, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26617.782473237476, "interactivity_tok_s": 35.54036743836254, "output_per_gpu": 67.312699085183, "total_per_gpu": 607.4802610718572, "tpot_error_pct": 33.9, "tpot_ms": 28.137019172193266, "tpot_relative": 2.621422, "ttft_error_pct": 5.12, "ttft_ms": 864.9327064249992, "ttft_relative": 1.652547, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 20150.551316404017, "interactivity_tok_s": 47.58921275789911, "output_per_gpu": 88.86711845108091, "total_per_gpu": 802.0035008408921, "tpot_ms": 21.01316542232598, "tpot_relative": 1.957719, "ttft_ms": 911.5730659221299, "ttft_relative": 1.741659, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44654.54983456234, "interactivity_tok_s": 23.377540918795322, "output_per_gpu": 91.63679882923861, "total_per_gpu": null, "tpot_error_pct": 25.67, "tpot_ms": 42.77609879814217, "tpot_relative": 3.98529, "ttft_error_pct": 34.05, "ttft_ms": 894.6007640629048, "ttft_relative": 1.709231, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 39911.42541902511, "interactivity_tok_s": 23.960874753759676, "output_per_gpu": 91.1621575983524, "total_per_gpu": 815.2010399122884, "tpot_error_pct": 22.61, "tpot_ms": 41.73470335606554, "tpot_relative": 3.888268, "ttft_error_pct": 6.89, "ttft_ms": 1263.0234570687492, "ttft_relative": 2.413143, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 32880.23251387349, "interactivity_tok_s": 29.378949508019193, "output_per_gpu": 110.71918547465765, "total_per_gpu": 990.0862102765075, "tpot_ms": 34.037976740013896, "tpot_relative": 3.171192, "ttft_ms": 1356.4207244926365, "ttft_relative": 2.591588, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71208.72590152484, "interactivity_tok_s": 14.563876836459096, "output_per_gpu": 114.9297350344075, "total_per_gpu": null, "tpot_error_pct": 17.94, "tpot_ms": 68.66303603286508, "tpot_relative": 6.39708, "ttft_error_pct": 55.12, "ttft_ms": 966.4400399038667, "ttft_relative": 1.846488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 64852.29883478619, "interactivity_tok_s": 14.691223862411235, "output_per_gpu": 112.3368393774526, "total_per_gpu": 1012.5793286648386, "tpot_error_pct": 16.91, "tpot_ms": 68.06784849004897, "tpot_relative": 6.341629, "ttft_error_pct": 4.21, "ttft_ms": 2062.6558921124993, "ttft_relative": 3.940927, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696219581", "measured": {"e2e_ms": 55854.702514722885, "interactivity_tok_s": 17.17604192636981, "output_per_gpu": 130.45228358076173, "total_per_gpu": 1175.867920648617, "tpot_ms": 58.22063105614181, "tpot_relative": 5.4242, "ttft_ms": 2153.2590451468423, "ttft_relative": 4.114034, "unavailable_metrics": {}}, "status": "success"}], "precision": "int4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["int4"], "rows": 20}]}, {"aic": {"coverage_pct": 100.0, "points": 81, "status_counts": {"failed": 0, "success": 81, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.04, "tpot_shape_error_pct": 10.82, "ttft_mape_pct": 53.99, "ttft_shape_error_pct": 41.48}, "aisimulate": {"coverage_pct": 100.0, "points": 81, "status_counts": {"failed": 0, "success": 81, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.24, "tpot_shape_error_pct": 10.81, "ttft_mape_pct": 58.45, "ttft_shape_error_pct": 33.45}, "frameworks": ["sglang", "trt"], "gpu_skus": ["b200", "b300"], "hf_model_paths": ["nvidia/DeepSeek-R1-0528-FP4-V2"], "model": "nvidia/DeepSeek-R1-0528-FP4-V2", "precisions": ["fp4"], "rows": 81, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 36, "status_counts": {"failed": 0, "success": 36, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.05, "tpot_shape_error_pct": 11.82, "ttft_mape_pct": 57.0, "ttft_shape_error_pct": 40.57}, "aisimulate": {"coverage_pct": 100.0, "points": 36, "status_counts": {"failed": 0, "success": 36, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.66, "tpot_shape_error_pct": 12.24, "ttft_mape_pct": 73.69, "ttft_shape_error_pct": 34.88}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 20, "status_counts": {"failed": 0, "success": 20, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.28, "tpot_shape_error_pct": 10.54, "ttft_mape_pct": 56.16, "ttft_shape_error_pct": 25.57}, "aisimulate": {"coverage_pct": 100.0, "points": 20, "status_counts": {"failed": 0, "success": 20, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.68, "tpot_shape_error_pct": 11.18, "ttft_mape_pct": 70.89, "ttft_shape_error_pct": 21.02}, "gpu": "b200", "precisions": ["fp4"], "rows": 20, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.9, "tpot_shape_error_pct": null, "ttft_mape_pct": 67.92, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.57, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.05, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "98b7abda7682ecbe", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6606.054447113909, "interactivity_tok_s": 156.07920625621216, "output_per_gpu": 19.429255580953765, "total_per_gpu": null, "tpot_error_pct": 9.9, "tpot_ms": 6.4070033669856565, "tpot_relative": 1.098986, "ttft_error_pct": 67.92, "ttft_ms": 51.69000268758325, "ttft_relative": 0.320759, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5905.264967899997, "interactivity_tok_s": 156.54911370244713, "output_per_gpu": 19.50378190073967, "total_per_gpu": 38.869974717091665, "tpot_error_pct": 9.57, "tpot_ms": 6.387771711698731, "tpot_relative": 1.095687, "ttft_error_pct": 84.05, "ttft_ms": 25.707806799999208, "ttft_relative": 0.159528, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 5527.156207399958, "interactivity_tok_s": 171.5288618722849, "output_per_gpu": 20.83603153986908, "total_per_gpu": 41.52507566383937, "tpot_ms": 5.829922667734887, "tpot_relative": 1.0, "ttft_ms": 161.14896959998077, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.66, "tpot_shape_error_pct": 8.36, "ttft_mape_pct": 67.95, "ttft_shape_error_pct": 30.43}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.56, "tpot_shape_error_pct": 11.59, "ttft_mape_pct": 79.85, "ttft_shape_error_pct": 15.8}, "framework": "sglang", "id": "da30dbf2daa0072e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5770.108425934598, "interactivity_tok_s": 179.41393745123528, "output_per_gpu": 44.572845995947866, "total_per_gpu": null, "tpot_error_pct": 8.4, "tpot_ms": 5.573702992120108, "tpot_relative": 0.916006, "ttft_error_pct": 56.07, "ttft_ms": 68.21026499572767, "ttft_relative": 0.439299, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5145.8130284000035, "interactivity_tok_s": 180.05830031890002, "output_per_gpu": 44.7645491059793, "total_per_gpu": 89.21330749219642, "tpot_error_pct": 8.73, "tpot_ms": 5.553756745614653, "tpot_relative": 0.912727, "ttft_error_pct": 78.18, "ttft_ms": 33.873329600000424, "ttft_relative": 0.218157, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 5755.645939001261, "interactivity_tok_s": 164.3441554188252, "output_per_gpu": 40.01869464659941, "total_per_gpu": 79.75507811976394, "tpot_ms": 6.084791986983265, "tpot_relative": 1.0, "ttft_ms": 155.27068940064055, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6140.808108288915, "interactivity_tok_s": 168.56221730575393, "output_per_gpu": 83.77263015117839, "total_per_gpu": null, "tpot_error_pct": 15.45, "tpot_ms": 5.932527561535966, "tpot_relative": 0.974976, "ttft_error_pct": 61.76, "ttft_ms": 71.83241283762169, "ttft_relative": 0.462627, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5564.789199049999, "interactivity_tok_s": 168.97429762826908, "output_per_gpu": 83.77517109657413, "total_per_gpu": 166.01860690766156, "tpot_error_pct": 15.65, "tpot_ms": 5.918059811675772, "tpot_relative": 0.972599, "ttft_error_pct": 81.05, "ttft_ms": 35.59378775000003, "ttft_relative": 0.229237, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 6744.101269900034, "interactivity_tok_s": 142.52438380822508, "output_per_gpu": 69.16205780530328, "total_per_gpu": 137.0595647541824, "tpot_ms": 7.016343261975149, "tpot_relative": 1.153095, "ttft_ms": 187.82654205003837, "ttft_relative": 1.209672, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8531.915192243436, "interactivity_tok_s": 120.97515964715221, "output_per_gpu": 119.64306859258609, "total_per_gpu": null, "tpot_error_pct": 2.71, "tpot_ms": 8.26615978781674, "tpot_relative": 1.358495, "ttft_error_pct": 62.97, "ttft_ms": 75.6337293069095, "ttft_relative": 0.487109, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7619.043292150008, "interactivity_tok_s": 120.94131315303413, "output_per_gpu": 117.50575246602293, "total_per_gpu": 236.16066631305057, "tpot_error_pct": 2.74, "tpot_ms": 8.268473145604442, "tpot_relative": 1.358875, "ttft_error_pct": 81.02, "ttft_ms": 38.765913275000734, "ttft_relative": 0.249667, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 7584.048908600005, "interactivity_tok_s": 124.25327853376078, "output_per_gpu": 117.56167439541326, "total_per_gpu": 236.2730570669428, "tpot_ms": 8.048077377115572, "tpot_relative": 1.322654, "ttft_ms": 204.25049112503757, "ttft_relative": 1.315448, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10383.785584357562, "interactivity_tok_s": 99.31458214781514, "output_per_gpu": 196.89405193864116, "total_per_gpu": null, "tpot_error_pct": 2.53, "tpot_ms": 10.069014825150722, "tpot_relative": 1.654784, "ttft_error_pct": 60.57, "ttft_ms": 83.1834182283732, "ttft_relative": 0.535732, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9436.93902353744, "interactivity_tok_s": 98.8530819675134, "output_per_gpu": 193.60085277209555, "total_per_gpu": 385.77169628535995, "tpot_error_pct": 3.01, "tpot_ms": 10.116022486063056, "tpot_relative": 1.662509, "ttft_error_pct": 78.76, "ttft_ms": 44.808221099999706, "ttft_relative": 0.288581, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 9330.34284598757, "interactivity_tok_s": 101.82722002960665, "output_per_gpu": 195.08813326727304, "total_per_gpu": 388.73527165840943, "tpot_ms": 9.8205568187882, "tpot_relative": 1.613951, "ttft_ms": 210.9783657126627, "ttft_relative": 1.358778, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12911.234744331576, "interactivity_tok_s": 79.84247170355184, "output_per_gpu": 317.22908527486186, "total_per_gpu": null, "tpot_error_pct": 1.52, "tpot_ms": 12.524662359062646, "tpot_relative": 2.058355, "ttft_error_pct": 71.6, "ttft_ms": 98.50515101049022, "ttft_relative": 0.634409, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11632.567912756258, "interactivity_tok_s": 79.02714366100233, "output_per_gpu": 308.83736290426975, "total_per_gpu": 620.9624044157105, "tpot_error_pct": 2.57, "tpot_ms": 12.653880093270685, "tpot_relative": 2.079591, "ttft_error_pct": 84.85, "ttft_ms": 52.54415398749997, "ttft_relative": 0.338404, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 11640.536427356528, "interactivity_tok_s": 81.05674190804237, "output_per_gpu": 308.39032037459884, "total_per_gpu": 620.0635604368277, "tpot_ms": 12.337036703677095, "tpot_relative": 2.02752, "ttft_ms": 346.8594648064027, "ttft_relative": 2.233902, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15935.300563597686, "interactivity_tok_s": 64.72406995683004, "output_per_gpu": 515.1226927011705, "total_per_gpu": null, "tpot_error_pct": 0.32, "tpot_ms": 15.45020269378895, "tpot_relative": 2.539151, "ttft_error_pct": 69.35, "ttft_ms": 129.74320785158972, "ttft_relative": 0.835594, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14581.76644857497, "interactivity_tok_s": 63.58467864735512, "output_per_gpu": 495.85273424595175, "total_per_gpu": 990.1369566236848, "tpot_error_pct": 1.46, "tpot_ms": 15.72705911664769, "tpot_relative": 2.58465, "ttft_error_pct": 84.0, "ttft_ms": 67.74847911562486, "ttft_relative": 0.436325, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 14733.266412849996, "interactivity_tok_s": 64.51614074519692, "output_per_gpu": 490.8981861323661, "total_per_gpu": 980.2435329278541, "tpot_ms": 15.49999718596695, "tpot_relative": 2.547334, "ttft_ms": 423.3208459499991, "ttft_relative": 2.726341, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19939.119169301943, "interactivity_tok_s": 51.70253053441082, "output_per_gpu": 824.0769447550362, "total_per_gpu": null, "tpot_error_pct": 1.94, "tpot_ms": 19.341413073281707, "tpot_relative": 3.178648, "ttft_error_pct": 69.86, "ttft_ms": 152.8535953347572, "ttft_relative": 0.984433, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18532.1432311499, "interactivity_tok_s": 49.97504188337437, "output_per_gpu": 776.9390233690268, "total_per_gpu": 1554.2283508771682, "tpot_error_pct": 1.45, "tpot_ms": 20.009988232399635, "tpot_relative": 3.288525, "ttft_error_pct": 80.7, "ttft_ms": 97.90799027031206, "ttft_relative": 0.630563, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28554749907", "measured": {"e2e_ms": 18677.564261367166, "interactivity_tok_s": 50.69759826599148, "output_per_gpu": 768.5752187515458, "total_per_gpu": 1537.4969705928854, "tpot_ms": 19.724800270682866, "tpot_relative": 3.241656, "ttft_ms": 507.1647465375235, "ttft_relative": 3.266326, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25052.403356966814, "interactivity_tok_s": 41.09728980004769, "output_per_gpu": 1311.3453531929897, "total_per_gpu": null, "tpot_error_pct": 2.54, "tpot_ms": 24.332504767719247, "tpot_relative": 3.998905, "ttft_error_pct": 77.06, "ttft_ms": 160.25097959002508, "ttft_relative": 1.032075, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23926.862209507093, "interactivity_tok_s": 38.68252035300011, "output_per_gpu": 1200.3074268820417, "total_per_gpu": 2403.2517361320024, "tpot_error_pct": 3.55, "tpot_ms": 25.851469626964025, "tpot_relative": 4.248538, "ttft_error_pct": 77.6, "ttft_ms": 156.48183432890588, "ttft_relative": 1.0078, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28554749907", "measured": {"e2e_ms": 23648.041234221862, "interactivity_tok_s": 40.054045251468004, "output_per_gpu": 1213.4806596851895, "total_per_gpu": 2429.6271412120773, "tpot_ms": 24.96626729514541, "tpot_relative": 4.10306, "ttft_ms": 698.6637594288965, "ttft_relative": 4.49965, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31692.407072730588, "interactivity_tok_s": 32.45889362953189, "output_per_gpu": 2072.7589700390085, "total_per_gpu": null, "tpot_error_pct": 6.55, "tpot_ms": 30.80819732839494, "tpot_relative": 5.063147, "ttft_error_pct": 82.34, "ttft_ms": 175.62120578256554, "ttft_relative": 1.131065, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31231.38976094144, "interactivity_tok_s": 29.784228415493114, "output_per_gpu": 1846.2831533522885, "total_per_gpu": 3690.78188002537, "tpot_error_pct": 1.85, "tpot_ms": 33.5748163776444, "tpot_relative": 5.517825, "ttft_error_pct": 72.47, "ttft_ms": 273.87874233398526, "ttft_relative": 1.763879, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28554749907", "measured": {"e2e_ms": 31381.06798012461, "interactivity_tok_s": 30.33390269161776, "output_per_gpu": 1835.7023311523292, "total_per_gpu": 3669.6304619557877, "tpot_ms": 32.96641418568051, "tpot_relative": 5.417837, "ttft_ms": 994.6785645492305, "ttft_relative": 6.406094, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.14, "tpot_shape_error_pct": 0.59, "ttft_mape_pct": 97.75, "ttft_shape_error_pct": 2.56}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.08, "tpot_shape_error_pct": 1.57, "ttft_mape_pct": 98.36, "ttft_shape_error_pct": 54.6}, "framework": "sglang", "id": "881199132c0bd723", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 18772.952629367883, "interactivity_tok_s": 54.98364612958501, "output_per_gpu": 875.541185227291, "total_per_gpu": null, "tpot_error_pct": 23.91, "tpot_ms": 18.187226027957625, "tpot_relative": 0.760876, "ttft_error_pct": 97.77, "ttft_ms": 167.42040276722992, "ttft_relative": 0.022257, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17342.163820815782, "interactivity_tok_s": 53.42399209411179, "output_per_gpu": 830.7599938715463, "total_per_gpu": 1661.8945585339918, "tpot_error_pct": 21.69, "tpot_ms": 18.718181865525857, "tpot_relative": 0.783089, "ttft_error_pct": 98.72, "ttft_ms": 96.61199370937518, "ttft_relative": 0.012844, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506952", "measured": {"e2e_ms": 29556.46698904675, "interactivity_tok_s": 41.83575526588148, "output_per_gpu": 489.17051435373577, "total_per_gpu": 978.5615845693303, "tpot_ms": 23.902998610748995, "tpot_relative": 1.0, "ttft_ms": 7522.079313911672, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23701.673093369074, "interactivity_tok_s": 43.48085463643209, "output_per_gpu": 1386.4044889193997, "total_per_gpu": null, "tpot_error_pct": 24.36, "tpot_ms": 22.998627979177574, "tpot_relative": 0.962165, "ttft_error_pct": 97.72, "ttft_ms": 174.07667067041652, "ttft_relative": 0.023142, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22391.939704393775, "interactivity_tok_s": 41.347415734926166, "output_per_gpu": 1284.043462893923, "total_per_gpu": 2570.907762759373, "tpot_error_pct": 20.46, "tpot_ms": 24.18530837358476, "tpot_relative": 1.011811, "ttft_error_pct": 98.01, "ttft_ms": 151.42335808749993, "ttft_relative": 0.020131, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506952", "measured": {"e2e_ms": 35591.06822982612, "interactivity_tok_s": 32.88680193230149, "output_per_gpu": 811.086223503432, "total_per_gpu": 1623.9542729905404, "tpot_ms": 30.407334895576998, "tpot_relative": 1.272114, "ttft_ms": 7625.748287561146, "ttft_relative": 1.013782, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.99, "tpot_shape_error_pct": null, "ttft_mape_pct": 40.6, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.01, "tpot_shape_error_pct": null, "ttft_mape_pct": 65.41, "ttft_shape_error_pct": null}, "framework": "trt", "id": "d995697b5d166a67", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6477.852864667926, "interactivity_tok_s": 160.17597478606777, "output_per_gpu": 78.60045034772233, "total_per_gpu": null, "tpot_error_pct": 2.99, "tpot_ms": 6.24313353694652, "tpot_relative": 0.970117, "ttft_error_pct": 40.6, "ttft_ms": 91.12725637163568, "ttft_relative": 0.594007, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5834.411247674997, "interactivity_tok_s": 158.57916288618466, "output_per_gpu": 76.27299506211583, "total_per_gpu": 153.29191088555174, "tpot_error_pct": 2.01, "tpot_ms": 6.305998731483527, "tpot_relative": 0.979886, "ttft_error_pct": 65.41, "ttft_ms": 53.06353677499982, "ttft_relative": 0.345891, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 6053.7243307006065, "interactivity_tok_s": 155.3894363148057, "output_per_gpu": 73.63965559427311, "total_per_gpu": 147.99947889560292, "tpot_ms": 6.435443899636045, "tpot_relative": 1.0, "ttft_ms": 153.4111074755856, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.62, "tpot_shape_error_pct": null, "ttft_mape_pct": 19.02, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.61, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.9, "ttft_shape_error_pct": null}, "framework": "trt", "id": "e97d3539eb8d2fe1", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 15079.760285118264, "interactivity_tok_s": 68.60278427751268, "output_per_gpu": 545.2855635960007, "total_per_gpu": null, "tpot_error_pct": 6.62, "tpot_ms": 14.5766678500218, "tpot_relative": 0.933799, "ttft_error_pct": 19.02, "ttft_ms": 167.82907454596338, "ttft_relative": 1.190194, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13976.423988887236, "interactivity_tok_s": 66.46258810191385, "output_per_gpu": 513.5560014176306, "total_per_gpu": 1027.3435535587448, "tpot_error_pct": 3.61, "tpot_ms": 15.0460586708811, "tpot_relative": 0.963868, "ttft_error_pct": 20.9, "ttft_ms": 111.53386051718765, "ttft_relative": 0.790965, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 14521.829442425224, "interactivity_tok_s": 64.06118246305113, "output_per_gpu": 492.7167935883385, "total_per_gpu": 985.6557419752052, "tpot_ms": 15.610077141126371, "tpot_relative": 1.0, "ttft_ms": 141.0098649939755, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.51, "tpot_shape_error_pct": 49.6, "ttft_mape_pct": 60.16, "ttft_shape_error_pct": 25.04}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 71.98, "tpot_shape_error_pct": 30.88, "ttft_mape_pct": 69.28, "ttft_shape_error_pct": 30.75}, "framework": "trt", "id": "46bde6b8f24703ea", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 18338.46528663545, "interactivity_tok_s": 56.58672636490318, "output_per_gpu": 892.5989059896269, "total_per_gpu": null, "tpot_error_pct": 13.25, "tpot_ms": 17.67198889632588, "tpot_relative": 0.867497, "ttft_error_pct": 64.59, "ttft_ms": 260.0206456940765, "ttft_relative": 0.354104, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28125.497242635185, "interactivity_tok_s": 32.950167650364044, "output_per_gpu": 511.18173248695837, "total_per_gpu": 1023.4864490253432, "tpot_error_pct": 48.98, "tpot_ms": 30.3488592413566, "tpot_relative": 1.489789, "ttft_error_pct": 73.37, "ttft_ms": 195.51536265859335, "ttft_relative": 0.266259, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 19460.124712144898, "interactivity_tok_s": 49.08879227867226, "output_per_gpu": 727.2741297144415, "total_per_gpu": 1456.1459637222483, "tpot_ms": 20.37124878369584, "tpot_relative": 1.0, "ttft_ms": 734.3053470784071, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30885.37204863349, "interactivity_tok_s": 33.49395496528785, "output_per_gpu": 1063.5139071845126, "total_per_gpu": null, "tpot_error_pct": 29.78, "tpot_ms": 29.856133772090235, "tpot_relative": 1.465602, "ttft_error_pct": 55.72, "ttft_ms": 342.547199785178, "ttft_relative": 0.466491, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41666.59278144925, "interactivity_tok_s": 22.29234776293533, "output_per_gpu": 694.5764848922337, "total_per_gpu": 1388.4816638647112, "tpot_error_pct": 94.99, "tpot_ms": 44.85844248593069, "tpot_relative": 2.202047, "ttft_error_pct": 65.19, "ttft_ms": 269.3320123667971, "ttft_relative": 0.366785, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 21974.671845796547, "interactivity_tok_s": 43.46683627394792, "output_per_gpu": 1287.5614785239713, "total_per_gpu": 2573.8785330552164, "tpot_ms": 23.006045199552638, "tpot_relative": 1.129339, "ttft_ms": 773.6551952812079, "ttft_relative": 1.053588, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.27, "tpot_shape_error_pct": null, "ttft_mape_pct": 39.37, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 81.16, "tpot_shape_error_pct": null, "ttft_mape_pct": 48.01, "ttft_shape_error_pct": null}, "framework": "trt", "id": "7ac933725e1686f5", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 43502.31001022957, "interactivity_tok_s": 23.72577591871908, "output_per_gpu": 1511.2927430102836, "total_per_gpu": null, "tpot_error_pct": 39.27, "tpot_ms": 42.14825274527791, "tpot_relative": 1.392692, "ttft_error_pct": 39.37, "ttft_ms": 384.64745181026933, "ttft_relative": 0.606315, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 50898.37495258854, "interactivity_tok_s": 18.23983770234486, "output_per_gpu": 1137.2702797237232, "total_per_gpu": 2273.441391410884, "tpot_error_pct": 81.16, "tpot_ms": 54.82504923119151, "tpot_relative": 1.811567, "ttft_error_pct": 48.01, "ttft_ms": 329.85401572578286, "ttft_relative": 0.519945, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 28524.98650017169, "interactivity_tok_s": 33.04268771916967, "output_per_gpu": 1991.492969288144, "total_per_gpu": 3981.061167080974, "tpot_ms": 30.263881936572957, "tpot_relative": 1.0, "ttft_ms": 634.4021987129963, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.51, "tpot_shape_error_pct": 4.68, "ttft_mape_pct": 9.63, "ttft_shape_error_pct": 17.86}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.36, "tpot_shape_error_pct": 4.48, "ttft_mape_pct": 48.51, "ttft_shape_error_pct": 20.24}, "framework": "trt", "id": "ee5d2b774ecc58b0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7769.668226956184, "interactivity_tok_s": 133.91177091392865, "output_per_gpu": 130.9000922575209, "total_per_gpu": null, "tpot_error_pct": 1.44, "tpot_ms": 7.4676034315366255, "tpot_relative": 0.985643, "ttft_error_pct": 19.24, "ttft_ms": 130.3099164942155, "ttft_relative": 0.807577, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6992.372845875012, "interactivity_tok_s": 132.5045566335274, "output_per_gpu": 127.72267586763465, "total_per_gpu": 256.6944307250669, "tpot_error_pct": 0.39, "tpot_ms": 7.546910275438572, "tpot_relative": 0.996111, "ttft_error_pct": 54.63, "ttft_ms": 73.21056292500005, "ttft_relative": 0.453712, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 7108.694815519266, "interactivity_tok_s": 131.98925464513547, "output_per_gpu": 125.56570287591423, "total_per_gpu": 252.35939036956395, "tpot_ms": 7.576374324475022, "tpot_relative": 1.0, "ttft_ms": 161.359184270259, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9078.1231771056, "interactivity_tok_s": 114.49613438442852, "output_per_gpu": 224.62707845854595, "total_per_gpu": null, "tpot_error_pct": 6.11, "tpot_ms": 8.733919318554742, "tpot_relative": 1.152784, "ttft_error_pct": 7.09, "ttft_ms": 143.32371422409778, "ttft_relative": 0.888228, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8292.743599899968, "interactivity_tok_s": 113.09600170260808, "output_per_gpu": 219.88831044292965, "total_per_gpu": 438.15244250368926, "tpot_error_pct": 4.95, "tpot_ms": 8.84204556257924, "tpot_relative": 1.167055, "ttft_error_pct": 46.15, "ttft_ms": 83.06607615000011, "ttft_relative": 0.51479, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 8792.053805314936, "interactivity_tok_s": 107.4954975838385, "output_per_gpu": 207.4817834782994, "total_per_gpu": 413.4310278837362, "tpot_ms": 9.302715206467827, "tpot_relative": 1.227858, "ttft_ms": 154.2601879540598, "ttft_relative": 0.956005, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11373.461017964371, "interactivity_tok_s": 91.30635593168357, "output_per_gpu": 359.770004899321, "total_per_gpu": null, "tpot_error_pct": 5.98, "tpot_ms": 10.95214007607763, "tpot_relative": 1.445565, "ttft_error_pct": 2.55, "ttft_ms": 169.4217201369549, "ttft_relative": 1.049966, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10252.531795524994, "interactivity_tok_s": 90.11737338829731, "output_per_gpu": 350.3767730219189, "total_per_gpu": 704.4834257782106, "tpot_error_pct": 4.74, "tpot_ms": 11.096639442554597, "tpot_relative": 1.464637, "ttft_error_pct": 44.74, "ttft_ms": 96.06717090624974, "ttft_relative": 0.595362, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 10838.841674188734, "interactivity_tok_s": 85.84362440448669, "output_per_gpu": 331.66870536429343, "total_per_gpu": 666.8681367296125, "tpot_ms": 11.64908875804333, "tpot_relative": 1.537555, "ttft_ms": 173.85605943854898, "ttft_relative": 1.077448, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 16, "status_counts": {"failed": 0, "success": 16, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.26, "tpot_shape_error_pct": 12.92, "ttft_mape_pct": 58.05, "ttft_shape_error_pct": 53.43}, "aisimulate": {"coverage_pct": 100.0, "points": 16, "status_counts": {"failed": 0, "success": 16, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.39, "tpot_shape_error_pct": 13.15, "ttft_mape_pct": 77.19, "ttft_shape_error_pct": 46.76}, "gpu": "b300", "precisions": ["fp4"], "rows": 16, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.7, "tpot_shape_error_pct": 15.12, "ttft_mape_pct": 53.46, "ttft_shape_error_pct": 53.76}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.41, "tpot_shape_error_pct": 17.41, "ttft_mape_pct": 75.24, "ttft_shape_error_pct": 49.34}, "framework": "sglang", "id": "dd7cbea057bae439", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5382.796477971085, "interactivity_tok_s": 192.48077216200875, "output_per_gpu": 47.798418169823606, "total_per_gpu": null, "tpot_error_pct": 12.5, "tpot_ms": 5.19532412909437, "tpot_relative": 0.875012, "ttft_error_pct": 12.13, "ttft_ms": 67.97989390754324, "ttft_relative": 0.878723, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4811.809617500085, "interactivity_tok_s": 192.63043514426533, "output_per_gpu": 47.871802567217, "total_per_gpu": 95.40589435009831, "tpot_error_pct": 12.57, "tpot_ms": 5.191287655302638, "tpot_relative": 0.874332, "ttft_error_pct": 56.43, "ttft_ms": 33.70630109999884, "ttft_relative": 0.435695, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 5542.216343100881, "interactivity_tok_s": 168.42292488806305, "output_per_gpu": 41.56036579367388, "total_per_gpu": 82.82754472207874, "tpot_ms": 5.9374339963791645, "tpot_relative": 1.0, "ttft_ms": 77.36211740411818, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5838.398454038997, "interactivity_tok_s": 177.3911908039411, "output_per_gpu": 88.13535462033512, "total_per_gpu": null, "tpot_error_pct": 16.04, "tpot_ms": 5.637258510233661, "tpot_relative": 0.949444, "ttft_error_pct": 31.08, "ttft_ms": 71.48299806996279, "ttft_relative": 0.924005, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5294.124021399999, "interactivity_tok_s": 177.66087811435258, "output_per_gpu": 88.06528681451225, "total_per_gpu": 174.52039837692274, "tpot_error_pct": 16.17, "tpot_ms": 5.628701212184392, "tpot_relative": 0.948002, "ttft_error_pct": 65.98, "ttft_ms": 35.28686250000068, "ttft_relative": 0.456126, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6377.25327477674, "interactivity_tok_s": 148.92915564826325, "output_per_gpu": 73.13988385495925, "total_per_gpu": 144.94248675411012, "tpot_ms": 6.714601957200189, "tpot_relative": 1.130893, "ttft_ms": 103.71422133757733, "ttft_relative": 1.340633, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7854.64086616006, "interactivity_tok_s": 131.50155456020124, "output_per_gpu": 129.91613029902211, "total_per_gpu": null, "tpot_error_pct": 1.49, "tpot_ms": 7.604472839461386, "tpot_relative": 1.280768, "ttft_error_pct": 38.96, "ttft_ms": 75.26515139106206, "ttft_relative": 0.972894, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7009.301872649981, "interactivity_tok_s": 131.51552971860806, "output_per_gpu": 127.65160288217258, "total_per_gpu": 256.55158968748935, "tpot_error_pct": 1.5, "tpot_ms": 7.603664769777454, "tpot_relative": 1.280631, "ttft_error_pct": 68.86, "ttft_ms": 38.401232349999525, "ttft_relative": 0.496383, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7201.1860892991535, "interactivity_tok_s": 129.5429590356851, "output_per_gpu": 123.7405118051714, "total_per_gpu": 248.69115855648832, "tpot_ms": 7.719446949830217, "tpot_relative": 1.300132, "ttft_ms": 123.29997536726296, "ttft_relative": 1.593803, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10028.834108070154, "interactivity_tok_s": 102.8552807386489, "output_per_gpu": 203.84584750220174, "total_per_gpu": null, "tpot_error_pct": 5.67, "tpot_ms": 9.722398235837394, "tpot_relative": 1.637475, "ttft_error_pct": 35.12, "ttft_ms": 82.82071280850006, "ttft_relative": 1.070559, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9115.143045862505, "interactivity_tok_s": 102.35049881685967, "output_per_gpu": 200.48154996991934, "total_per_gpu": 399.4822672442344, "tpot_error_pct": 6.19, "tpot_ms": 9.770348083885207, "tpot_relative": 1.645551, "ttft_error_pct": 65.58, "ttft_ms": 43.940814350000174, "ttft_relative": 0.567989, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8669.60677300085, "interactivity_tok_s": 108.68595973633913, "output_per_gpu": 209.84031058671403, "total_per_gpu": 418.1306611256371, "tpot_ms": 9.200820441075336, "tpot_relative": 1.549629, "ttft_ms": 127.66122489992995, "ttft_relative": 1.650177, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12140.53579066146, "interactivity_tok_s": 84.94834892402096, "output_per_gpu": 337.3474989148901, "total_per_gpu": null, "tpot_error_pct": 2.38, "tpot_ms": 11.771859166967618, "tpot_relative": 1.982651, "ttft_error_pct": 61.79, "ttft_ms": 97.92386285358809, "ttft_relative": 1.265786, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10926.461043774972, "interactivity_tok_s": 84.14866880036256, "output_per_gpu": 328.6341011751554, "total_per_gpu": 660.7666239592123, "tpot_error_pct": 3.35, "tpot_ms": 11.88372928836744, "tpot_relative": 2.001492, "ttft_error_pct": 80.01, "ttft_ms": 51.23868219999986, "ttft_relative": 0.662323, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 10778.545780737506, "interactivity_tok_s": 86.96724112665018, "output_per_gpu": 332.8160539944708, "total_per_gpu": 669.1750479057672, "tpot_ms": 11.498582535735524, "tpot_relative": 1.936625, "ttft_ms": 256.29184023124196, "ttft_relative": 3.312886, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15199.120029429341, "interactivity_tok_s": 67.88093230398995, "output_per_gpu": 540.0953148414304, "total_per_gpu": null, "tpot_error_pct": 1.3, "tpot_ms": 14.731677454306581, "tpot_relative": 2.481152, "ttft_error_pct": 82.27, "ttft_ms": 128.6139936737091, "ttft_relative": 1.662493, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13899.618799018737, "interactivity_tok_s": 66.71008604400221, "output_per_gpu": 520.21971133679, "total_per_gpu": 1038.7938316846469, "tpot_error_pct": 3.08, "tpot_ms": 14.990236998651095, "tpot_relative": 2.5247, "ttft_error_pct": 90.98, "ttft_ms": 65.42457100625003, "ttft_relative": 0.845693, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 14146.488825618144, "interactivity_tok_s": 68.76393735272468, "output_per_gpu": 511.26493655945524, "total_per_gpu": 1020.9126084243493, "tpot_ms": 14.54250641394339, "tpot_relative": 2.449291, "ttft_ms": 725.2313224373211, "ttft_relative": 9.374502, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20080.830368853167, "interactivity_tok_s": 51.329792340908284, "output_per_gpu": 818.2303971678992, "total_per_gpu": null, "tpot_error_pct": 4.54, "tpot_ms": 19.481863346698763, "tpot_relative": 3.281192, "ttft_error_pct": 81.9, "ttft_ms": 150.88416518033586, "ttft_relative": 1.950362, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18682.184308981163, "interactivity_tok_s": 49.55947866787613, "output_per_gpu": 771.8201598045048, "total_per_gpu": 1543.9883157689387, "tpot_error_pct": 8.27, "tpot_ms": 20.177774804725463, "tpot_relative": 3.3984, "ttft_error_pct": 88.76, "ttft_ms": 93.69866335156213, "ttft_relative": 1.21117, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 18000.917305830626, "interactivity_tok_s": 53.65844371993583, "output_per_gpu": 797.5552949518882, "total_per_gpu": 1595.4701894509622, "tpot_ms": 18.636395889888025, "tpot_relative": 3.138796, "ttft_ms": 833.4582370145654, "ttft_relative": 10.773467, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24804.377080199814, "interactivity_tok_s": 41.50662583038456, "output_per_gpu": 1324.445192046024, "total_per_gpu": null, "tpot_error_pct": 1.66, "tpot_ms": 24.092538962007335, "tpot_relative": 4.057736, "ttft_error_pct": 84.45, "ttft_ms": 157.7097220663093, "ttft_relative": 2.038591, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23713.045647640512, "interactivity_tok_s": 39.02049565588669, "output_per_gpu": 1211.8116823715322, "total_per_gpu": 2426.285520110031, "tpot_error_pct": 8.14, "tpot_ms": 25.62755759995423, "tpot_relative": 4.316268, "ttft_error_pct": 85.33, "ttft_ms": 148.74866974843727, "ttft_relative": 1.922759, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 22799.718550262496, "interactivity_tok_s": 42.196030277205494, "output_per_gpu": 1258.4560617270001, "total_per_gpu": 2519.6767490204593, "tpot_ms": 23.69891180356378, "tpot_relative": 3.99144, "ttft_ms": 1013.9932500476548, "ttft_relative": 13.107103, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.82, "tpot_shape_error_pct": 10.72, "ttft_mape_pct": 62.64, "ttft_shape_error_pct": 53.1}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.38, "tpot_shape_error_pct": 8.89, "ttft_mape_pct": 79.15, "ttft_shape_error_pct": 44.18}, "framework": "sglang", "id": "5400d619172b5927", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6283.712072197233, "interactivity_tok_s": 164.19488092201047, "output_per_gpu": 20.432344078698932, "total_per_gpu": null, "tpot_error_pct": 8.01, "tpot_ms": 6.090323854097385, "tpot_relative": 1.080144, "ttft_error_pct": 30.22, "ttft_ms": 53.31076945560824, "ttft_relative": 0.697794, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5628.753298099998, "interactivity_tok_s": 164.2762431862619, "output_per_gpu": 20.461902289957823, "total_per_gpu": 40.77945645219183, "tpot_error_pct": 7.96, "tpot_ms": 6.0873074560523435, "tpot_relative": 1.079609, "ttft_error_pct": 66.01, "ttft_ms": 25.966205699999705, "ttft_relative": 0.339876, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 5265.956434281543, "interactivity_tok_s": 177.3541753412055, "output_per_gpu": 21.870744217926674, "total_per_gpu": 43.58720165767175, "tpot_ms": 5.6384350584142435, "tpot_relative": 1.0, "ttft_ms": 76.39902161899954, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6500.365547131554, "interactivity_tok_s": 158.7009285771983, "output_per_gpu": 39.50464541923457, "total_per_gpu": null, "tpot_error_pct": 0.15, "tpot_ms": 6.301160358450965, "tpot_relative": 1.117537, "ttft_error_pct": 47.92, "ttft_ms": 54.27850043621683, "ttft_relative": 0.710461, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5916.651768350019, "interactivity_tok_s": 158.64570058880477, "output_per_gpu": 39.39398703129033, "total_per_gpu": 78.06769907916973, "tpot_error_pct": 0.18, "tpot_ms": 6.3033539282095585, "tpot_relative": 1.117926, "ttft_error_pct": 73.56, "ttft_ms": 27.557857099999758, "ttft_relative": 0.36071, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 5982.986243895721, "interactivity_tok_s": 158.9357373610792, "output_per_gpu": 38.977559512507554, "total_per_gpu": 77.24245795293398, "tpot_ms": 6.291851138099568, "tpot_relative": 1.115886, "ttft_ms": 104.22689240076579, "ttft_relative": 1.364244, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7199.033911673874, "interactivity_tok_s": 143.23874663710285, "output_per_gpu": 70.92800884665323, "total_per_gpu": null, "tpot_error_pct": 0.77, "tpot_ms": 6.981351229870172, "tpot_relative": 1.238172, "ttft_error_pct": 51.33, "ttft_ms": 57.111603516688476, "ttft_relative": 0.747544, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6463.729974775009, "interactivity_tok_s": 142.51029397753936, "output_per_gpu": 68.83873051562388, "total_per_gpu": 138.35067752461748, "tpot_error_pct": 1.28, "tpot_ms": 7.017036959853631, "tpot_relative": 1.244501, "ttft_error_pct": 73.8, "ttft_ms": 30.74540082500058, "ttft_relative": 0.402432, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6469.173865678022, "interactivity_tok_s": 144.3412111678149, "output_per_gpu": 68.78582007452476, "total_per_gpu": 138.244339198513, "tpot_ms": 6.928028328911371, "tpot_relative": 1.228715, "ttft_ms": 117.34112013364211, "ttft_relative": 1.535898, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8399.573620325353, "interactivity_tok_s": 122.7107206446723, "output_per_gpu": 121.72427693150652, "total_per_gpu": null, "tpot_error_pct": 1.85, "tpot_ms": 8.149247227515298, "tpot_relative": 1.445303, "ttft_error_pct": 48.36, "ttft_ms": 62.8937065772032, "ttft_relative": 0.823227, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7635.705053062481, "interactivity_tok_s": 122.14205801659887, "output_per_gpu": 119.22080783957989, "total_per_gpu": 237.56100561667924, "tpot_error_pct": 2.33, "tpot_ms": 8.18718806804534, "tpot_relative": 1.452032, "ttft_error_pct": 71.09, "ttft_ms": 35.20304118750031, "ttft_relative": 0.460779, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7549.968801776413, "interactivity_tok_s": 124.9819666838395, "output_per_gpu": 120.3276240808854, "total_per_gpu": 239.7664627351313, "tpot_ms": 8.001154298760948, "tpot_relative": 1.419038, "ttft_ms": 121.78355984797236, "ttft_relative": 1.594046, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9188.835031296176, "interactivity_tok_s": 112.24028113052144, "output_per_gpu": 222.8551707286553, "total_per_gpu": null, "tpot_error_pct": 6.76, "tpot_ms": 8.909457370630824, "tpot_relative": 1.580129, "ttft_error_pct": 66.71, "ttft_ms": 74.46014114084481, "ttft_relative": 0.974622, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8307.793375937526, "interactivity_tok_s": 110.70195193777002, "output_per_gpu": 215.5217086774923, "total_per_gpu": 433.337719133554, "tpot_error_pct": 5.47, "tpot_ms": 9.033264386902047, "tpot_relative": 1.602087, "ttft_error_pct": 81.41, "ttft_ms": 41.57122351250001, "ttft_relative": 0.544133, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8969.744351762347, "interactivity_tok_s": 104.64753486760117, "output_per_gpu": 199.70926620981572, "total_per_gpu": 401.5445053783351, "tpot_ms": 9.555886827770843, "tpot_relative": 1.694776, "ttft_ms": 223.643510810507, "ttft_relative": 2.927309, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11387.479954434477, "interactivity_tok_s": 90.61339409883787, "output_per_gpu": 360.44425283019353, "total_per_gpu": null, "tpot_error_pct": 4.62, "tpot_ms": 11.035896071934305, "tpot_relative": 1.957262, "ttft_error_pct": 85.28, "ttft_ms": 97.75827284568398, "ttft_relative": 1.279575, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10450.962280725005, "interactivity_tok_s": 88.76699361886236, "output_per_gpu": 344.88857242913554, "total_per_gpu": 688.6861721123191, "tpot_error_pct": 2.64, "tpot_ms": 11.265448555052867, "tpot_relative": 1.997974, "ttft_error_pct": 91.86, "ttft_ms": 54.042208031249764, "ttft_relative": 0.707368, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 11343.941111219101, "interactivity_tok_s": 86.42697338721939, "output_per_gpu": 318.20676614318813, "total_per_gpu": 635.4069610712309, "tpot_ms": 11.570461868654046, "tpot_relative": 2.05207, "ttft_ms": 664.0113531517272, "ttft_relative": 8.691359, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14558.616629927885, "interactivity_tok_s": 70.82938314616946, "output_per_gpu": 564.354819850639, "total_per_gpu": null, "tpot_error_pct": 4.55, "tpot_ms": 14.118434406471055, "tpot_relative": 2.503963, "ttft_error_pct": 84.84, "ttft_ms": 115.45823210799638, "ttft_relative": 1.511253, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13597.515574517229, "interactivity_tok_s": 68.14926069915715, "output_per_gpu": 528.0274623239616, "total_per_gpu": 1056.2930002240682, "tpot_error_pct": 0.79, "tpot_ms": 14.673673488762697, "tpot_relative": 2.602437, "ttft_error_pct": 89.66, "ttft_ms": 78.75433231718765, "ttft_relative": 1.030829, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 14386.860460619937, "interactivity_tok_s": 67.60983461829983, "output_per_gpu": 497.83287277946613, "total_per_gpu": 995.8902070812351, "tpot_ms": 14.790747613060008, "tpot_relative": 2.623201, "ttft_ms": 761.6301900612598, "ttft_relative": 9.969109, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17114.570805103678, "interactivity_tok_s": 60.19711908778947, "output_per_gpu": 959.9298429319721, "total_per_gpu": null, "tpot_error_pct": 11.82, "tpot_ms": 16.61209066403383, "tpot_relative": 2.946224, "ttft_error_pct": 86.51, "ttft_ms": 120.40205579707235, "ttft_relative": 1.575963, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16350.169435969477, "interactivity_tok_s": 56.6843887822581, "output_per_gpu": 876.333019313598, "total_per_gpu": 1754.591201327435, "tpot_error_pct": 6.35, "tpot_ms": 17.641541551083893, "tpot_relative": 3.128801, "ttft_error_pct": 85.8, "ttft_ms": 126.67718088359388, "ttft_relative": 1.6581, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 18209.395994114067, "interactivity_tok_s": 53.084280171740325, "output_per_gpu": 786.9518423942621, "total_per_gpu": 1575.6324914185068, "tpot_ms": 18.83796854294268, "tpot_relative": 3.340992, "ttft_ms": 892.2586017218727, "ttft_relative": 11.678927, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 36}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.01, "tpot_shape_error_pct": 3.44, "ttft_mape_pct": 91.29, "ttft_shape_error_pct": 57.66}, "aisimulate": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.87, "tpot_shape_error_pct": 3.69, "ttft_mape_pct": 95.56, "ttft_shape_error_pct": 56.16}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.01, "tpot_shape_error_pct": 3.44, "ttft_mape_pct": 91.29, "ttft_shape_error_pct": 57.66}, "aisimulate": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.87, "tpot_shape_error_pct": 3.69, "ttft_mape_pct": 95.56, "ttft_shape_error_pct": 56.16}, "gpu": "b200", "precisions": ["fp4"], "rows": 6, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.01, "tpot_shape_error_pct": 3.44, "ttft_mape_pct": 91.29, "ttft_shape_error_pct": 57.66}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.87, "tpot_shape_error_pct": 3.69, "ttft_mape_pct": 95.56, "ttft_shape_error_pct": 56.16}, "framework": "sglang", "id": "ac83997ac9506590", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 64701.56952511748, "interactivity_tok_s": 126.76068357988117, "output_per_gpu": 126.55157587665536, "total_per_gpu": null, "tpot_error_pct": 21.19, "tpot_ms": 7.8888814083258465, "tpot_relative": 0.788098, "ttft_error_pct": 86.27, "ttft_ms": 83.74190952046645, "ttft_relative": 0.137343, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56983.59225024986, "interactivity_tok_s": 127.53374692891457, "output_per_gpu": 125.77303365201847, "total_per_gpu": 141.8227538116601, "tpot_error_pct": 21.67, "tpot_ms": 7.84106186856868, "tpot_relative": 0.783321, "ttft_error_pct": 93.19, "ttft_ms": 41.54793152499908, "ttft_relative": 0.068142, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 73305.49028561218, "interactivity_tok_s": 99.89986452081406, "output_per_gpu": 97.53621997535176, "total_per_gpu": 109.98268000401563, "tpot_ms": 10.010023585083548, "tpot_relative": 1.0, "ttft_ms": 609.7261561895721, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71609.1701877205, "interactivity_tok_s": 114.53225953303011, "output_per_gpu": 228.72719221099507, "total_per_gpu": null, "tpot_error_pct": 27.49, "tpot_ms": 8.731164512751175, "tpot_relative": 0.872242, "ttft_error_pct": 86.22, "ttft_ms": 92.20166377563854, "ttft_relative": 0.151218, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 63890.55197738714, "interactivity_tok_s": 115.23381565543752, "output_per_gpu": 225.04945638795942, "total_per_gpu": 253.25595568113434, "tpot_error_pct": 27.93, "tpot_ms": 8.678008224513853, "tpot_relative": 0.866932, "ttft_error_pct": 93.0, "ttft_ms": 46.8464599125014, "ttft_relative": 0.076832, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 89294.33482852764, "interactivity_tok_s": 83.04508090841087, "output_per_gpu": 160.7459739647396, "total_per_gpu": 180.89301752480446, "tpot_ms": 12.04165242614291, "tpot_relative": 1.202959, "ttft_ms": 668.9038831216749, "ttft_relative": 1.097056, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 90527.24534862912, "interactivity_tok_s": 90.59067101998079, "output_per_gpu": 361.95461325300033, "total_per_gpu": null, "tpot_error_pct": 23.8, "tpot_ms": 11.038664232649726, "tpot_relative": 1.102761, "ttft_error_pct": 81.59, "ttft_ms": 109.54661899520954, "ttft_relative": 0.179665, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 80795.51870892472, "interactivity_tok_s": 91.36157470330538, "output_per_gpu": 357.403861804397, "total_per_gpu": 402.24389594724835, "tpot_error_pct": 24.44, "tpot_ms": 10.945520622290903, "tpot_relative": 1.093456, "ttft_error_pct": 90.93, "ttft_ms": 53.96480468124603, "ttft_relative": 0.088507, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 107482.56813309563, "interactivity_tok_s": 69.02888787146127, "output_per_gpu": 267.8628093380836, "total_per_gpu": 301.4689865508335, "tpot_ms": 14.486688556566357, "tpot_relative": 1.447218, "ttft_ms": 595.1418789598392, "ttft_relative": 0.976081, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 117628.22551186781, "interactivity_tok_s": 69.72137383990885, "output_per_gpu": 557.3035078161209, "total_per_gpu": null, "tpot_error_pct": 23.19, "tpot_ms": 14.342804005786746, "tpot_relative": 1.432844, "ttft_error_pct": 98.14, "ttft_ms": 146.31790046858762, "ttft_relative": 0.239973, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 105008.25864115558, "interactivity_tok_s": 70.65922791288914, "output_per_gpu": 552.8694013688182, "total_per_gpu": 621.5137916378435, "tpot_error_pct": 24.21, "tpot_ms": 14.152433157532244, "tpot_relative": 1.413826, "ttft_error_pct": 99.15, "ttft_ms": 66.94699031249874, "ttft_relative": 0.109798, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 146372.17833073955, "interactivity_tok_s": 53.55538147594728, "output_per_gpu": 396.53879462121165, "total_per_gpu": 445.7731412994587, "tpot_ms": 18.67225986335507, "tpot_relative": 1.865356, "ttft_ms": 7876.4695524048875, "ttft_relative": 12.918044, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 155370.22369700827, "interactivity_tok_s": 52.77889205706136, "output_per_gpu": 843.9521306862989, "total_per_gpu": null, "tpot_error_pct": 19.87, "tpot_ms": 18.946968400148684, "tpot_relative": 1.8928, "ttft_error_pct": 97.59, "ttft_ms": 175.60553139039757, "ttft_relative": 0.288007, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 137938.5850670534, "interactivity_tok_s": 53.57709754586747, "output_per_gpu": 836.1118264181472, "total_per_gpu": 940.5079620507897, "tpot_error_pct": 21.06, "tpot_ms": 18.664691553025953, "tpot_relative": 1.8646, "ttft_error_pct": 98.71, "ttft_ms": 94.24548801562612, "ttft_relative": 0.15457, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 181924.7377831154, "interactivity_tok_s": 42.294200282515014, "output_per_gpu": 632.2440215561993, "total_per_gpu": 711.1890700106293, "tpot_ms": 23.643903734323906, "tpot_relative": 2.362023, "ttft_ms": 7288.333181756751, "ttft_relative": 11.953453, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 199853.66662512487, "interactivity_tok_s": 41.0238856546291, "output_per_gpu": 1312.1507209818376, "total_per_gpu": null, "tpot_error_pct": 22.5, "tpot_ms": 24.376042981856376, "tpot_relative": 2.435163, "ttft_error_pct": 97.94, "ttft_ms": 189.49856073931187, "ttft_relative": 0.310793, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 176404.33939961882, "interactivity_tok_s": 41.79317578815634, "output_per_gpu": 1304.1171618023757, "total_per_gpu": 1467.3487632845358, "tpot_error_pct": 23.92, "tpot_ms": 23.927351323308322, "tpot_relative": 2.390339, "ttft_error_pct": 98.39, "ttft_ms": 147.54118712031217, "ttft_relative": 0.241979, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216506943", "measured": {"e2e_ms": 240984.10281427103, "interactivity_tok_s": 31.79507760150178, "output_per_gpu": 953.6857571438171, "total_per_gpu": 1073.0597015345202, "tpot_ms": 31.451409319811408, "tpot_relative": 3.141992, "ttft_ms": 9184.281505796991, "ttft_relative": 15.062961, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:8192", "label": "1k8k", "precisions": ["fp4"], "rows": 6}, {"aic": {"coverage_pct": 100.0, "points": 39, "status_counts": {"failed": 0, "success": 39, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.81, "tpot_shape_error_pct": 11.21, "ttft_mape_pct": 45.46, "ttft_shape_error_pct": 39.43}, "aisimulate": {"coverage_pct": 100.0, "points": 39, "status_counts": {"failed": 0, "success": 39, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.53, "tpot_shape_error_pct": 10.75, "ttft_mape_pct": 38.67, "ttft_shape_error_pct": 28.07}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 26, "status_counts": {"failed": 0, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.54, "tpot_shape_error_pct": 11.05, "ttft_mape_pct": 47.31, "ttft_shape_error_pct": 35.79}, "aisimulate": {"coverage_pct": 100.0, "points": 26, "status_counts": {"failed": 0, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.95, "tpot_shape_error_pct": 10.1, "ttft_mape_pct": 38.16, "ttft_shape_error_pct": 24.88}, "gpu": "b200", "precisions": ["fp4"], "rows": 26, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.88, "tpot_shape_error_pct": null, "ttft_mape_pct": 48.31, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.57, "tpot_shape_error_pct": null, "ttft_mape_pct": 28.61, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "3cc1e4d7143d2ba7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7019.684719607428, "interactivity_tok_s": 151.5254352672811, "output_per_gpu": 18.552573145648303, "total_per_gpu": null, "tpot_error_pct": 9.88, "tpot_ms": 6.599552070159472, "tpot_relative": 1.098849, "ttft_error_pct": 48.31, "ttft_ms": 268.34295183428833, "ttft_relative": 1.4831, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6244.8469314, "interactivity_tok_s": 151.95581847816135, "output_per_gpu": 18.62135313762288, "total_per_gpu": 166.68943393407318, "tpot_error_pct": 9.57, "tpot_ms": 6.58086021328441, "tpot_relative": 1.095737, "ttft_error_pct": 28.61, "ttft_ms": 129.17708689999995, "ttft_relative": 0.713947, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 5762.38522019994, "interactivity_tok_s": 166.5035500694615, "output_per_gpu": 20.178925436110617, "total_per_gpu": 180.63207509594193, "tpot_ms": 6.005877950246843, "tpot_relative": 1.0, "ttft_ms": 180.9337703998608, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.53, "tpot_shape_error_pct": 4.81, "ttft_mape_pct": 28.82, "ttft_shape_error_pct": 37.94}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.29, "tpot_shape_error_pct": 4.77, "ttft_mape_pct": 45.51, "ttft_shape_error_pct": 29.37}, "framework": "sglang", "id": "e217b14fc01c87a8", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6300.458707205301, "interactivity_tok_s": 170.95079516802275, "output_per_gpu": 41.581028143322115, "total_per_gpu": null, "tpot_error_pct": 10.91, "tpot_ms": 5.849636434958539, "tpot_relative": 1.109071, "ttft_error_pct": 51.83, "ttft_ms": 316.28063424271556, "ttft_relative": 1.518266, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5562.748657799991, "interactivity_tok_s": 171.69906195392034, "output_per_gpu": 41.80936697074877, "total_per_gpu": 374.2574270510668, "tpot_error_pct": 10.42, "tpot_ms": 5.824143641905129, "tpot_relative": 1.104237, "ttft_error_pct": 27.85, "ttft_ms": 150.29115410000014, "ttft_relative": 0.721454, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 5109.899179195054, "interactivity_tok_s": 189.59653952575744, "output_per_gpu": 45.5122074568419, "total_per_gpu": 407.4034814765093, "tpot_ms": 5.274357868035593, "tpot_relative": 1.0, "ttft_ms": 208.31698323599994, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6807.004257124904, "interactivity_tok_s": 157.86168653435382, "output_per_gpu": 76.94195919999346, "total_per_gpu": null, "tpot_error_pct": 4.88, "tpot_ms": 6.334659295448362, "tpot_relative": 1.201029, "ttft_error_pct": 26.97, "ttft_ms": 326.6477978812301, "ttft_relative": 1.568032, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6050.461224250007, "interactivity_tok_s": 158.57503598156404, "output_per_gpu": 77.05977233118519, "total_per_gpu": 686.7156583861496, "tpot_error_pct": 4.41, "tpot_ms": 6.3061628446753755, "tpot_relative": 1.195627, "ttft_error_pct": 38.69, "ttft_ms": 157.7220545000003, "ttft_relative": 0.757125, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 5900.941794901155, "interactivity_tok_s": 165.57269809576042, "output_per_gpu": 79.04606993684583, "total_per_gpu": 704.4164849881258, "tpot_ms": 6.039643078242533, "tpot_relative": 1.145095, "ttft_ms": 257.2558253770694, "ttft_relative": 1.234925, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8684.923649147813, "interactivity_tok_s": 122.64224400575587, "output_per_gpu": 115.88338417681594, "total_per_gpu": null, "tpot_error_pct": 14.12, "tpot_ms": 8.153797315980844, "tpot_relative": 1.545932, "ttft_error_pct": 19.15, "ttft_ms": 343.5889948994106, "ttft_relative": 1.649357, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7869.266140900011, "interactivity_tok_s": 119.41890410928947, "output_per_gpu": 114.54789090405656, "total_per_gpu": 1029.7172244277035, "tpot_error_pct": 17.2, "tpot_ms": 8.373883577802914, "tpot_relative": 1.587659, "ttft_error_pct": 40.2, "ttft_ms": 172.45076484999922, "ttft_relative": 0.827829, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 6854.089583083987, "interactivity_tok_s": 139.9574767847523, "output_per_gpu": 131.20535355055898, "total_per_gpu": 1179.4578793362368, "tpot_ms": 7.145027353829413, "tpot_relative": 1.354672, "ttft_ms": 288.363069773186, "ttft_relative": 1.384251, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10414.728188485919, "interactivity_tok_s": 101.92417370568603, "output_per_gpu": 194.46714463841786, "total_per_gpu": null, "tpot_error_pct": 6.99, "tpot_ms": 9.81121517734917, "tpot_relative": 1.860172, "ttft_error_pct": 0.43, "ttft_ms": 377.8550620577177, "ttft_relative": 1.813847, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9439.86169632503, "interactivity_tok_s": 100.28614662593391, "output_per_gpu": 193.27453166050884, "total_per_gpu": 1719.2663175068212, "tpot_error_pct": 8.73, "tpot_ms": 9.971466983670114, "tpot_relative": 1.890556, "ttft_error_pct": 45.7, "ttft_ms": 206.07518306249958, "ttft_relative": 0.989239, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 8872.114095499273, "interactivity_tok_s": 109.04438669563406, "output_per_gpu": 205.53292485191392, "total_per_gpu": 1828.3103924797058, "tpot_ms": 9.170577507957484, "tpot_relative": 1.73871, "ttft_ms": 379.49341850471683, "ttft_relative": 1.821711, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14293.943574278233, "interactivity_tok_s": 73.8779653707361, "output_per_gpu": 285.6368820147827, "total_per_gpu": null, "tpot_error_pct": 4.78, "tpot_ms": 13.535835684994531, "tpot_relative": 2.566348, "ttft_error_pct": 31.3, "ttft_ms": 446.7836685288275, "ttft_relative": 2.14473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12746.198492612515, "interactivity_tok_s": 73.3337132757294, "output_per_gpu": 280.044967734317, "total_per_gpu": 2527.3357393649703, "tpot_error_pct": 5.56, "tpot_ms": 13.636292986284127, "tpot_relative": 2.585394, "ttft_error_pct": 58.47, "ttft_ms": 270.04201414375, "ttft_relative": 1.296303, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 12474.794477500836, "interactivity_tok_s": 77.40925230306263, "output_per_gpu": 286.38331732810406, "total_per_gpu": 2584.5377579785127, "tpot_ms": 12.918352396492995, "tpot_relative": 2.449275, "ttft_ms": 650.2954908210086, "ttft_relative": 3.121663, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20679.043969321832, "interactivity_tok_s": 50.91639657373347, "output_per_gpu": 397.99327387854885, "total_per_gpu": null, "tpot_error_pct": 3.52, "tpot_ms": 19.64003871624874, "tpot_relative": 3.723683, "ttft_error_pct": 43.25, "ttft_ms": 587.2843625993738, "ttft_relative": 2.819186, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18554.271110815498, "interactivity_tok_s": 50.95539688856524, "output_per_gpu": 390.9232018503294, "total_per_gpu": 3495.7597436240317, "tpot_error_pct": 3.44, "tpot_ms": 19.625006595217144, "tpot_relative": 3.720833, "ttft_error_pct": 62.17, "ttft_ms": 391.4721351906251, "ttft_relative": 1.879214, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34557177019", "measured": {"e2e_ms": 18598.49201501711, "interactivity_tok_s": 52.70659730688633, "output_per_gpu": 390.00002481330137, "total_per_gpu": 3487.504400612909, "tpot_ms": 18.97295691803929, "tpot_relative": 3.597207, "ttft_ms": 1034.7931340846117, "ttft_relative": 4.967397, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.31, "tpot_shape_error_pct": 14.38, "ttft_mape_pct": 41.9, "ttft_shape_error_pct": 49.08}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.99, "tpot_shape_error_pct": 15.56, "ttft_mape_pct": 42.5, "ttft_shape_error_pct": 32.93}, "framework": "sglang", "id": "6a851cc617e91c95", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6249.1242684516965, "interactivity_tok_s": 173.01538381750137, "output_per_gpu": 41.99646622794363, "total_per_gpu": null, "tpot_error_pct": 10.02, "tpot_ms": 5.779832856104932, "tpot_relative": 0.899814, "ttft_error_pct": 71.54, "ttft_ms": 336.35525665635083, "ttft_relative": 1.715401, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5505.960049499983, "interactivity_tok_s": 173.78187403169935, "output_per_gpu": 42.24058981705125, "total_per_gpu": 378.1175274217736, "tpot_error_pct": 10.42, "tpot_ms": 5.7543400632081525, "tpot_relative": 0.895845, "ttft_error_pct": 19.23, "ttft_ms": 158.3710115000009, "ttft_relative": 0.807687, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 6165.046137398167, "interactivity_tok_s": 155.68168460189312, "output_per_gpu": 37.721850722844735, "total_per_gpu": 337.6679394599181, "tpot_ms": 6.423363175682387, "tpot_relative": 1.0, "ttft_ms": 196.0797002990148, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6761.285574248212, "interactivity_tok_s": 159.49395650950837, "output_per_gpu": 77.59272601871015, "total_per_gpu": null, "tpot_error_pct": 13.23, "tpot_ms": 6.26983004174446, "tpot_relative": 0.976098, "ttft_error_pct": 39.39, "ttft_ms": 347.24944154362873, "ttft_relative": 1.770961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5996.365247350002, "interactivity_tok_s": 160.28313753843432, "output_per_gpu": 77.75564981505195, "total_per_gpu": 692.9169479829562, "tpot_error_pct": 13.66, "tpot_ms": 6.238959477320001, "tpot_relative": 0.971292, "ttft_error_pct": 33.24, "ttft_ms": 166.32659429999998, "ttft_relative": 0.84826, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 7001.407611350015, "interactivity_tok_s": 138.3914077116606, "output_per_gpu": 66.6150251160586, "total_per_gpu": 593.6376328024962, "tpot_ms": 7.225882130511358, "tpot_relative": 1.124938, "ttft_ms": 249.12476959999594, "ttft_relative": 1.270528, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9237.04490058222, "interactivity_tok_s": 115.30917653296056, "output_per_gpu": 108.95775617388055, "total_per_gpu": null, "tpot_error_pct": 3.84, "tpot_ms": 8.672336669702561, "tpot_relative": 1.350124, "ttft_error_pct": 32.6, "ttft_ms": 365.24448747649956, "ttft_relative": 1.862735, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8355.018713750025, "interactivity_tok_s": 112.45990578841845, "output_per_gpu": 108.0129261567303, "total_per_gpu": 970.9717887130827, "tpot_error_pct": 6.47, "tpot_ms": 8.892057956027417, "tpot_relative": 1.384331, "ttft_error_pct": 33.93, "ttft_ms": 181.98432139999994, "ttft_relative": 0.928114, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 7951.112251024551, "interactivity_tok_s": 119.7314396954893, "output_per_gpu": 112.99027405786453, "total_per_gpu": 1015.7151779218793, "tpot_ms": 8.352025186895615, "tpot_relative": 1.300257, "ttft_ms": 275.45666262476516, "ttft_relative": 1.40482, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11963.137391782597, "interactivity_tok_s": 88.4831758834949, "output_per_gpu": 169.44931936353126, "total_per_gpu": null, "tpot_error_pct": 7.64, "tpot_ms": 11.301583493303768, "tpot_relative": 1.75945, "ttft_error_pct": 15.54, "ttft_ms": 401.61747813284114, "ttft_relative": 2.048236, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10823.42316270001, "interactivity_tok_s": 87.3128476379249, "output_per_gpu": 168.77233612086707, "total_per_gpu": 1501.307959856953, "tpot_error_pct": 9.09, "tpot_ms": 11.453068214506882, "tpot_relative": 1.783033, "ttft_error_pct": 37.36, "ttft_ms": 217.72319157500016, "ttft_relative": 1.110381, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 10069.918669262552, "interactivity_tok_s": 95.24597665451701, "output_per_gpu": 180.98119290538637, "total_per_gpu": 1609.9113855879675, "tpot_ms": 10.499131145741423, "tpot_relative": 1.634522, "ttft_ms": 347.5854726250873, "ttft_relative": 1.772674, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16101.48958187743, "interactivity_tok_s": 65.46474023221089, "output_per_gpu": 253.63150296344253, "total_per_gpu": null, "tpot_error_pct": 6.25, "tpot_ms": 15.275398580256887, "tpot_relative": 2.3781, "ttft_error_pct": 21.39, "ttft_ms": 474.7568342746353, "ttft_relative": 2.421244, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14349.635552912523, "interactivity_tok_s": 65.05158253698006, "output_per_gpu": 249.06150226113718, "total_per_gpu": 2247.7177185404084, "tpot_error_pct": 6.92, "tpot_ms": 15.372416181136364, "tpot_relative": 2.393204, "ttft_error_pct": 52.67, "ttft_ms": 285.8361147999998, "ttft_relative": 1.457755, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 13759.932608018738, "interactivity_tok_s": 69.55614013419566, "output_per_gpu": 258.9151640639501, "total_per_gpu": 2336.6445499680344, "tpot_ms": 14.376875974869877, "tpot_relative": 2.238216, "ttft_ms": 603.9133326937431, "ttft_relative": 3.079938, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22685.083431343253, "interactivity_tok_s": 46.370548618914185, "output_per_gpu": 362.7562327445232, "total_per_gpu": null, "tpot_error_pct": 5.65, "tpot_ms": 21.5654123098321, "tpot_relative": 3.35734, "ttft_error_pct": 34.07, "ttft_ms": 623.6666383850118, "ttft_relative": 3.180679, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20354.566063056373, "interactivity_tok_s": 46.412426369443565, "output_per_gpu": 356.42309075333503, "total_per_gpu": 3187.2487651183233, "tpot_error_pct": 5.56, "tpot_ms": 21.545953922770295, "tpot_relative": 3.35431, "ttft_error_pct": 56.13, "ttft_ms": 415.0182832781249, "ttft_relative": 2.11658, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 19841.48081653438, "interactivity_tok_s": 48.99156677451286, "output_per_gpu": 364.6797185514728, "total_per_gpu": 3261.082159857241, "tpot_ms": 20.411676250375304, "tpot_relative": 3.177724, "ttft_ms": 945.9885839593782, "ttft_relative": 4.824511, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33175.88914847733, "interactivity_tok_s": 31.532628257924753, "output_per_gpu": 497.06738497258175, "total_per_gpu": null, "tpot_error_pct": 3.41, "tpot_ms": 31.713182669721824, "tpot_relative": 4.937162, "ttft_error_pct": 50.94, "ttft_ms": 733.3032773519033, "ttft_relative": 3.739823, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29913.7960174266, "interactivity_tok_s": 31.533135363734395, "output_per_gpu": 485.2270574090126, "total_per_gpu": 4373.728963393369, "tpot_error_pct": 3.41, "tpot_ms": 31.712672668448924, "tpot_relative": 4.937082, "ttft_error_pct": 54.94, "ttft_ms": 673.4851688343758, "ttft_relative": 3.434752, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 29776.78142835158, "interactivity_tok_s": 32.60733841312994, "output_per_gpu": 486.9236523773183, "total_per_gpu": 4389.0216937527375, "tpot_ms": 30.667943127714214, "tpot_relative": 4.774437, "ttft_ms": 1494.7722363171933, "ttft_relative": 7.623289, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 51173.10772147709, "interactivity_tok_s": 20.290867652039037, "output_per_gpu": 643.3272319844963, "total_per_gpu": null, "tpot_error_pct": 0.46, "tpot_ms": 49.283254770010274, "tpot_relative": 7.6725, "ttft_error_pct": 69.69, "ttft_ms": 756.3380917565776, "ttft_relative": 3.857299, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46525.134149388134, "interactivity_tok_s": 20.30640918542693, "output_per_gpu": 624.2711443091439, "total_per_gpu": 5635.970984399221, "tpot_error_pct": 0.38, "tpot_ms": 49.24553577486554, "tpot_relative": 7.666628, "ttft_error_pct": 52.52, "ttft_ms": 1184.806789064062, "ttft_relative": 6.042476, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28048542583", "measured": {"e2e_ms": 47664.745701384374, "interactivity_tok_s": 20.38341145774161, "output_per_gpu": 609.3477462491394, "total_per_gpu": 5501.241325305489, "tpot_ms": 49.05950125537993, "tpot_relative": 7.637666, "ttft_ms": 2495.4882773453137, "ttft_relative": 12.726908, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.28, "tpot_shape_error_pct": 1.64, "ttft_mape_pct": 93.21, "ttft_shape_error_pct": 22.83}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.32, "tpot_shape_error_pct": 1.49, "ttft_mape_pct": 91.87, "ttft_shape_error_pct": 31.22}, "framework": "sglang", "id": "c926b67c28571cbe", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 31087.198213433425, "interactivity_tok_s": 33.65575005690569, "output_per_gpu": 530.4817627062473, "total_per_gpu": null, "tpot_error_pct": 22.91, "tpot_ms": 29.712604779545362, "tpot_relative": 0.770878, "ttft_error_pct": 92.34, "ttft_ms": 691.2035239585215, "ttft_relative": 0.076639, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28037.137397790422, "interactivity_tok_s": 33.647978942058195, "output_per_gpu": 517.9799159973871, "total_per_gpu": 4668.95595878565, "tpot_error_pct": 22.89, "tpot_ms": 29.719467006383933, "tpot_relative": 0.771056, "ttft_error_pct": 92.97, "ttft_ms": 634.3636227515632, "ttft_relative": 0.070336, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216706955", "measured": {"e2e_ms": 44580.89908735492, "interactivity_tok_s": 25.944476924890527, "output_per_gpu": 326.14803323128007, "total_per_gpu": 2939.825958829428, "tpot_ms": 38.543848962344015, "tpot_relative": 1.0, "ttft_ms": 9018.998504499905, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48020.69593745204, "interactivity_tok_s": 21.625030158108814, "output_per_gpu": 685.5762853395148, "total_per_gpu": null, "tpot_error_pct": 21.65, "tpot_ms": 46.242710076638964, "tpot_relative": 1.199743, "ttft_error_pct": 94.09, "ttft_ms": 714.4035290503753, "ttft_relative": 0.079211, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43639.564026492226, "interactivity_tok_s": 21.65136385162652, "output_per_gpu": 665.8167176950543, "total_per_gpu": 6011.047821231627, "tpot_error_pct": 21.75, "tpot_ms": 46.18646690586546, "tpot_relative": 1.198284, "ttft_error_pct": 90.77, "ttft_ms": 1114.9498446515597, "ttft_relative": 0.123622, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19216706955", "measured": {"e2e_ms": 66467.10295389785, "interactivity_tok_s": 16.942990542836053, "output_per_gpu": 438.39494639688246, "total_per_gpu": 3957.865456578803, "tpot_ms": 59.021457721513436, "tpot_relative": 1.531281, "ttft_ms": 12079.973948494444, "ttft_relative": 1.339392, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.02, "tpot_shape_error_pct": null, "ttft_mape_pct": 21.0, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.43, "tpot_shape_error_pct": null, "ttft_mape_pct": 13.09, "ttft_shape_error_pct": null}, "framework": "trt", "id": "dd17189d37e67b38", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 23033.135814519464, "interactivity_tok_s": 45.860067498322444, "output_per_gpu": 357.460926320642, "total_per_gpu": null, "tpot_error_pct": 5.02, "tpot_ms": 21.805462890707258, "tpot_relative": 1.050199, "ttft_error_pct": 21.0, "ttft_ms": 726.1472773259424, "ttft_relative": 1.210018, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20785.181937650057, "interactivity_tok_s": 45.68282237874072, "output_per_gpu": 349.29751081764965, "total_per_gpu": 3123.529560498995, "tpot_error_pct": 5.43, "tpot_ms": 21.890066067051215, "tpot_relative": 1.054274, "ttft_error_pct": 13.09, "ttft_ms": 521.5861285093755, "ttft_relative": 0.869147, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 19825.122638181303, "interactivity_tok_s": 48.1622140854576, "output_per_gpu": 365.4210419042726, "total_per_gpu": 3267.7113093205116, "tpot_ms": 20.763165045228813, "tpot_relative": 1.0, "ttft_ms": 600.1128721906298, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.67, "tpot_shape_error_pct": null, "ttft_mape_pct": 81.41, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 4.91, "ttft_shape_error_pct": null}, "framework": "trt", "id": "94e8fac2dc4870d7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7486.019038628379, "interactivity_tok_s": 144.37513721210618, "output_per_gpu": 66.79571852591604, "total_per_gpu": null, "tpot_error_pct": 1.67, "tpot_ms": 6.926400343646896, "tpot_relative": 1.016682, "ttft_error_pct": 81.41, "ttft_ms": 400.3114870776045, "ttft_relative": 1.814091, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6890.379139125002, "interactivity_tok_s": 137.5901916840821, "output_per_gpu": 65.27918714690483, "total_per_gpu": 586.8209608338323, "tpot_error_pct": 6.68, "tpot_ms": 7.267959930574692, "tpot_relative": 1.066817, "ttft_error_pct": 4.91, "ttft_ms": 209.82936017500043, "ttft_relative": 0.950884, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 6482.134233275428, "interactivity_tok_s": 146.78355432164184, "output_per_gpu": 69.33604502489801, "total_per_gpu": 623.2896937022858, "tpot_ms": 6.8127523183471475, "tpot_relative": 1.0, "ttft_ms": 220.66775356652215, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.4, "tpot_shape_error_pct": 38.44, "ttft_mape_pct": 13.47, "ttft_shape_error_pct": 3.81}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 292.04, "tpot_shape_error_pct": 26.35, "ttft_mape_pct": 41.59, "ttft_shape_error_pct": 9.59}, "framework": "trt", "id": "8c7e7dd9fe1b7ed2", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 29269.37552559236, "interactivity_tok_s": 37.487897071858, "output_per_gpu": 555.2689075990302, "total_per_gpu": null, "tpot_error_pct": 0.15, "tpot_ms": 26.67527597195351, "tpot_relative": 1.001527, "ttft_error_pct": 11.78, "ttft_ms": 1980.5682062839176, "ttft_relative": 0.882153, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 86318.65675207874, "interactivity_tok_s": 10.838507884764942, "output_per_gpu": 169.29613707594694, "total_per_gpu": 1528.4193815923177, "tpot_error_pct": 246.4, "tpot_ms": 92.26362250523816, "tpot_relative": 3.46405, "ttft_error_pct": 44.26, "ttft_ms": 1251.4388058164054, "ttft_relative": 0.557396, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 26748.058892020345, "interactivity_tok_s": 37.545128770661876, "output_per_gpu": 533.340702027454, "total_per_gpu": 4815.04350926286, "tpot_ms": 26.63461367008041, "tpot_relative": 1.0, "ttft_ms": 2245.1532942250196, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 53748.00219750394, "interactivity_tok_s": 19.995676429095035, "output_per_gpu": 614.0622814842503, "total_per_gpu": null, "tpot_error_pct": 38.66, "tpot_ms": 50.01081126442583, "tpot_relative": 1.877662, "ttft_error_pct": 15.15, "ttft_ms": 2586.9422739963115, "ttft_relative": 1.152234, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 147805.25955955658, "interactivity_tok_s": 6.3345704225837345, "output_per_gpu": 198.60918590955683, "total_per_gpu": 1786.9687445487511, "tpot_error_pct": 337.68, "tpot_ms": 157.86390130494777, "tpot_relative": 5.927021, "ttft_error_pct": 38.91, "ttft_ms": 1862.3110237257729, "ttft_relative": 0.829481, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 36333.534974072696, "interactivity_tok_s": 27.725148126417462, "output_per_gpu": 791.965779588281, "total_per_gpu": 7125.642695705471, "tpot_ms": 36.068337505009254, "tpot_relative": 1.35419, "ttft_ms": 3048.6806552130474, "ttft_relative": 1.357894, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.91, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.03, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 148.7, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.09, "ttft_shape_error_pct": null}, "framework": "trt", "id": "cda7e23100109057", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 77712.32444715733, "interactivity_tok_s": 13.565438893975177, "output_per_gpu": 850.3976201671476, "total_per_gpu": null, "tpot_error_pct": 30.91, "tpot_ms": 73.71674501767357, "tpot_relative": 1.309091, "ttft_error_pct": 36.03, "ttft_ms": 2300.0942940772625, "ttft_relative": 0.639692, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 131721.19058841275, "interactivity_tok_s": 7.140429129273054, "output_per_gpu": 444.8307997450043, "total_per_gpu": 4002.326136712178, "tpot_error_pct": 148.7, "tpot_ms": 140.04760524831468, "tpot_relative": 2.48702, "ttft_error_pct": 35.09, "ttft_ms": 2333.907613694928, "ttft_relative": 0.649096, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 55580.36635577601, "interactivity_tok_s": 17.758387738824986, "output_per_gpu": 1041.0159471921909, "total_per_gpu": 9366.449752512402, "tpot_ms": 56.31141828341263, "tpot_relative": 1.0, "ttft_ms": 3595.6301899044774, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.36, "tpot_shape_error_pct": 7.67, "ttft_mape_pct": 80.48, "ttft_shape_error_pct": 16.17}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.49, "tpot_shape_error_pct": 3.72, "ttft_mape_pct": 7.64, "ttft_shape_error_pct": 1.6}, "framework": "trt", "id": "3d307249e42081fb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8619.922456190157, "interactivity_tok_s": 126.2373608834735, "output_per_gpu": 115.68123584329545, "total_per_gpu": null, "tpot_error_pct": 0.71, "tpot_ms": 7.921585123464951, "tpot_relative": 0.992929, "ttft_error_pct": 105.39, "ttft_ms": 516.1408748855126, "ttft_relative": 2.053851, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7936.124083699999, "interactivity_tok_s": 119.85874428145144, "output_per_gpu": 113.47564678191885, "total_per_gpu": 1020.0783892414553, "tpot_error_pct": 4.58, "tpot_ms": 8.343154318818886, "tpot_relative": 1.04577, "ttft_error_pct": 6.37, "ttft_ms": 267.3058637499997, "ttft_relative": 1.063676, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 7583.960909489542, "interactivity_tok_s": 125.34473619934576, "output_per_gpu": 118.73431209834112, "total_per_gpu": 1067.3506542398222, "tpot_ms": 7.9779975635324645, "tpot_relative": 1.0, "ttft_ms": 251.3039122102782, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11102.404142719291, "interactivity_tok_s": 97.10207889895514, "output_per_gpu": 181.5303857670995, "total_per_gpu": null, "tpot_error_pct": 5.24, "tpot_ms": 10.298440685709773, "tpot_relative": 1.290855, "ttft_error_pct": 87.4, "ttft_ms": 567.0993212381953, "ttft_relative": 2.256628, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10142.029439749978, "interactivity_tok_s": 94.35514217507249, "output_per_gpu": 179.96490748904668, "total_per_gpu": 1600.8710569410698, "tpot_error_pct": 8.3, "tpot_ms": 10.598256512024928, "tpot_relative": 1.328436, "ttft_error_pct": 6.98, "ttft_ms": 323.736433912499, "ttft_relative": 1.288227, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 9366.796224340214, "interactivity_tok_s": 102.18583708782569, "output_per_gpu": 194.67448838797034, "total_per_gpu": 1731.719579852423, "tpot_ms": 9.78609197222243, "tpot_relative": 1.226635, "ttft_ms": 302.61298562109005, "ttft_relative": 1.204171, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15355.551593804506, "interactivity_tok_s": 69.65858351875833, "output_per_gpu": 265.4508556136577, "total_per_gpu": null, "tpot_error_pct": 5.02, "tpot_ms": 14.355732624547418, "tpot_relative": 1.799416, "ttft_error_pct": 72.35, "ttft_ms": 669.6371188924995, "ttft_relative": 2.664651, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13764.593805206285, "interactivity_tok_s": 68.55981832231163, "output_per_gpu": 259.7285748203257, "total_per_gpu": 2343.985378449988, "tpot_error_pct": 6.7, "tpot_ms": 14.585802945084044, "tpot_relative": 1.828254, "ttft_error_pct": 7.05, "ttft_ms": 415.94065949375016, "ttft_relative": 1.65513, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 12897.35259967565, "interactivity_tok_s": 73.15246897924295, "output_per_gpu": 276.70427917723833, "total_per_gpu": 2497.1868613019205, "tpot_ms": 13.670078590016566, "tpot_relative": 1.713472, "ttft_ms": 388.53895208030735, "ttft_relative": 1.546092, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23305.919782308283, "interactivity_tok_s": 45.611844151368054, "output_per_gpu": 353.5570124616823, "total_per_gpu": null, "tpot_error_pct": 10.47, "tpot_ms": 21.924129984338872, "tpot_relative": 2.748074, "ttft_error_pct": 56.8, "ttft_ms": 877.5348083296172, "ttft_relative": 3.491927, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20905.551343212497, "interactivity_tok_s": 45.63912202548918, "output_per_gpu": 347.7204142672089, "total_per_gpu": 3109.426660986395, "tpot_error_pct": 10.4, "tpot_ms": 21.9110262340609, "tpot_relative": 2.746432, "ttft_error_pct": 10.17, "ttft_ms": 616.5632680093754, "ttft_relative": 2.453457, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016880548", "measured": {"e2e_ms": 18929.733753473556, "interactivity_tok_s": 50.38611693206016, "output_per_gpu": 382.7074377165589, "total_per_gpu": 3422.291764783162, "tpot_ms": 19.846736777679933, "tpot_relative": 2.487684, "ttft_ms": 559.6523069958494, "ttft_relative": 2.226994, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.35, "tpot_shape_error_pct": 11.46, "ttft_mape_pct": 41.76, "ttft_shape_error_pct": 45.05}, "aisimulate": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.69, "tpot_shape_error_pct": 11.74, "ttft_mape_pct": 39.69, "ttft_shape_error_pct": 33.0}, "gpu": "b300", "precisions": ["fp4"], "rows": 13, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.62, "tpot_shape_error_pct": 14.33, "ttft_mape_pct": 40.56, "ttft_shape_error_pct": 49.85}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.73, "tpot_shape_error_pct": 15.73, "ttft_mape_pct": 45.79, "ttft_shape_error_pct": 34.68}, "framework": "sglang", "id": "ab8e75dfdc80969f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5769.793351824093, "interactivity_tok_s": 187.22281364101534, "output_per_gpu": 45.466822892707256, "total_per_gpu": null, "tpot_error_pct": 12.38, "tpot_ms": 5.341229418319818, "tpot_relative": 0.876152, "ttft_error_pct": 62.44, "ttft_ms": 305.7156568829195, "ttft_relative": 1.624434, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5096.453721600004, "interactivity_tok_s": 187.7378318452703, "output_per_gpu": 45.634673187414776, "total_per_gpu": 408.49973603731627, "tpot_error_pct": 12.63, "tpot_ms": 5.326576908719067, "tpot_relative": 0.873749, "ttft_error_pct": 22.18, "ttft_ms": 146.45717330000062, "ttft_relative": 0.778207, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 5853.472722100196, "interactivity_tok_s": 164.03572593068128, "output_per_gpu": 39.72942978399966, "total_per_gpu": 355.6388256145712, "tpot_ms": 6.096232965875879, "tpot_relative": 1.0, "ttft_ms": 188.19828150026296, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6434.5771717473835, "interactivity_tok_s": 167.18424926006887, "output_per_gpu": 81.43823950046969, "total_per_gpu": null, "tpot_error_pct": 15.62, "tpot_ms": 5.9814247121115915, "tpot_relative": 0.981167, "ttft_error_pct": 29.42, "ttft_ms": 315.5796912572247, "ttft_relative": 1.676847, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5719.238110299999, "interactivity_tok_s": 167.9019753172951, "output_per_gpu": 81.5307410583504, "total_per_gpu": 726.5585509903967, "tpot_error_pct": 15.98, "tpot_ms": 5.955856076798597, "tpot_relative": 0.976973, "ttft_error_pct": 36.89, "ttft_ms": 153.87150799999972, "ttft_relative": 0.817603, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6867.5441781873815, "interactivity_tok_s": 141.07368254506238, "output_per_gpu": 67.88736104476985, "total_per_gpu": 604.9760131083475, "tpot_ms": 7.08849433827302, "tpot_relative": 1.162766, "ttft_ms": 243.83236023131758, "ttft_relative": 1.295614, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8450.382711420156, "interactivity_tok_s": 126.01110628757793, "output_per_gpu": 119.11431706673831, "total_per_gpu": null, "tpot_error_pct": 2.38, "tpot_ms": 7.935808433566456, "tpot_relative": 1.301756, "ttft_error_pct": 30.11, "ttft_ms": 332.05068388167007, "ttft_relative": 1.764366, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7654.946390424986, "interactivity_tok_s": 122.77137645191674, "output_per_gpu": 117.81064562799652, "total_per_gpu": 1059.0474435335088, "tpot_error_pct": 0.19, "tpot_ms": 8.145221051518053, "tpot_relative": 1.336107, "ttft_error_pct": 34.03, "ttft_ms": 168.37064847499937, "ttft_relative": 0.894645, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 7726.816258625104, "interactivity_tok_s": 123.00679220894484, "output_per_gpu": 116.30161435788607, "total_per_gpu": 1045.4821523808828, "tpot_ms": 8.129632372668944, "tpot_relative": 1.33355, "ttft_ms": 255.20887512507218, "ttft_relative": 1.356064, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11363.307627508864, "interactivity_tok_s": 93.01478744810834, "output_per_gpu": 178.47965994372692, "total_per_gpu": null, "tpot_error_pct": 5.12, "tpot_ms": 10.7509787146252, "tpot_relative": 1.763545, "ttft_error_pct": 6.58, "ttft_ms": 365.0564024472833, "ttft_relative": 1.939744, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10310.377110875002, "interactivity_tok_s": 91.59607270406987, "output_per_gpu": 177.17143921362543, "total_per_gpu": 1576.0218651013934, "tpot_error_pct": 6.74, "tpot_ms": 10.917498648995759, "tpot_relative": 1.79086, "ttft_error_pct": 41.27, "ttft_ms": 201.15049919999942, "ttft_relative": 1.068822, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 9813.251833739923, "interactivity_tok_s": 97.77278408799198, "output_per_gpu": 185.6015587031589, "total_per_gpu": 1651.011675535243, "tpot_ms": 10.227795079457245, "tpot_relative": 1.677724, "ttft_ms": 342.52161316398997, "ttft_relative": 1.820004, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15153.139077777063, "interactivity_tok_s": 69.49206308397409, "output_per_gpu": 269.5400064133197, "total_per_gpu": null, "tpot_error_pct": 2.77, "tpot_ms": 14.390132565090228, "tpot_relative": 2.360496, "ttft_error_pct": 27.41, "ttft_ms": 432.0334636897594, "ttft_relative": 2.295629, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13533.6787089188, "interactivity_tok_s": 68.93978723447127, "output_per_gpu": 264.047110151533, "total_per_gpu": 2382.9590788973574, "tpot_error_pct": 3.59, "tpot_ms": 14.505411752996825, "tpot_relative": 2.379406, "ttft_error_pct": 55.78, "ttft_ms": 263.14545273125043, "ttft_relative": 1.398235, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 13407.935854302195, "interactivity_tok_s": 71.41577294450694, "output_per_gpu": 265.99190930317894, "total_per_gpu": 2400.5104044634168, "tpot_ms": 14.002508952427666, "tpot_relative": 2.296912, "ttft_ms": 595.1331257776474, "ttft_relative": 3.162267, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21499.223490769262, "interactivity_tok_s": 48.874227199061146, "output_per_gpu": 382.71207293768384, "total_per_gpu": null, "tpot_error_pct": 2.09, "tpot_ms": 20.460681576141823, "tpot_relative": 3.356283, "ttft_error_pct": 39.83, "ttft_ms": 567.9462383761802, "ttft_relative": 3.017808, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19352.904946621667, "interactivity_tok_s": 48.77765008619479, "output_per_gpu": 374.859600270405, "total_per_gpu": 3352.113903533387, "tpot_error_pct": 2.29, "tpot_ms": 20.50119262065524, "tpot_relative": 3.362928, "ttft_error_pct": 59.64, "ttft_ms": 380.97921027187493, "ttft_relative": 2.02435, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 19494.627062215626, "interactivity_tok_s": 49.89561870033, "output_per_gpu": 371.4066846469489, "total_per_gpu": 3321.2368325959874, "tpot_ms": 20.04183986585953, "tpot_relative": 3.287578, "ttft_ms": 943.8410180843192, "ttft_relative": 5.015142, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32374.68759363482, "interactivity_tok_s": 32.26508052225891, "output_per_gpu": 509.1839261866756, "total_per_gpu": null, "tpot_error_pct": 3.7, "tpot_ms": 30.99325908423268, "tpot_relative": 5.084002, "ttft_error_pct": 55.97, "ttft_ms": 668.5835504647886, "ttft_relative": 3.552549, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29372.88333617195, "interactivity_tok_s": 32.061108072964224, "output_per_gpu": 494.38868352100195, "total_per_gpu": 4456.309827065223, "tpot_error_pct": 4.36, "tpot_ms": 31.190437888928038, "tpot_relative": 5.116346, "ttft_error_pct": 59.36, "ttft_ms": 617.0032626078118, "ttft_relative": 3.278474, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 29080.189502346184, "interactivity_tok_s": 33.457744568536256, "output_per_gpu": 498.7352762090318, "total_per_gpu": 4495.489048506877, "tpot_ms": 29.88844624453265, "tpot_relative": 4.902773, "ttft_ms": 1518.3294814527471, "ttft_relative": 8.067712, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48882.322409870496, "interactivity_tok_s": 21.22813108864424, "output_per_gpu": 673.3693700169857, "total_per_gpu": null, "tpot_error_pct": 0.92, "tpot_ms": 47.10730284376938, "tpot_relative": 7.727281, "ttft_error_pct": 72.69, "ttft_ms": 691.5516006944166, "ttft_relative": 3.67459, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44830.47131662667, "interactivity_tok_s": 21.044890601595004, "output_per_gpu": 647.8235878954367, "total_per_gpu": 5848.604372749959, "tpot_error_pct": 0.06, "tpot_ms": 47.51747200454487, "tpot_relative": 7.794563, "ttft_error_pct": 57.18, "ttft_ms": 1084.530936452346, "ttft_relative": 5.762704, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 46311.056270274225, "interactivity_tok_s": 21.032709964661358, "output_per_gpu": 627.1433686413218, "total_per_gpu": 5661.901660747804, "tpot_ms": 47.54499071589802, "tpot_relative": 7.799077, "ttft_ms": 2532.53568837187, "ttft_relative": 13.456742, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.32, "tpot_shape_error_pct": 6.44, "ttft_mape_pct": 43.7, "ttft_shape_error_pct": 36.66}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.03, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 29.92, "ttft_shape_error_pct": 30.07}, "framework": "sglang", "id": "2aa47bfd42a7c13d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6628.626056152668, "interactivity_tok_s": 160.54824169703386, "output_per_gpu": 19.65183232097576, "total_per_gpu": null, "tpot_error_pct": 6.25, "tpot_ms": 6.228657439220495, "tpot_relative": 1.06246, "ttft_error_pct": 91.48, "ttft_ms": 256.7094958301014, "ttft_relative": 1.914756, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5894.842344699995, "interactivity_tok_s": 161.02776302134572, "output_per_gpu": 19.72699068102358, "total_per_gpu": 176.5865716384951, "tpot_error_pct": 5.93, "tpot_ms": 6.2101092459903375, "tpot_relative": 1.059296, "ttft_error_pct": 7.73, "ttft_ms": 123.71111479999963, "ttft_relative": 0.922742, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 5582.126206438988, "interactivity_tok_s": 170.57604349865792, "output_per_gpu": 20.831052297584115, "total_per_gpu": 186.46960240068955, "tpot_ms": 5.862487952523462, "tpot_relative": 1.0, "ttft_ms": 134.0690063778311, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6961.5357606602165, "interactivity_tok_s": 152.77106449905773, "output_per_gpu": 37.43240467857394, "total_per_gpu": null, "tpot_error_pct": 0.16, "tpot_ms": 6.545742174927163, "tpot_relative": 1.116547, "ttft_error_pct": 52.15, "ttft_ms": 265.2415157097283, "ttft_relative": 1.978395, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6229.531269200001, "interactivity_tok_s": 153.19604852984568, "output_per_gpu": 37.41868433121853, "total_per_gpu": 333.45539013560466, "tpot_error_pct": 0.43, "tpot_ms": 6.527583508821246, "tpot_relative": 1.113449, "ttft_error_pct": 25.37, "ttft_ms": 130.0994401500003, "ttft_relative": 0.970392, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6299.259009840898, "interactivity_tok_s": 152.52985277680742, "output_per_gpu": 37.02027288521225, "total_per_gpu": 329.90495947410557, "tpot_ms": 6.556093655077944, "tpot_relative": 1.118313, "ttft_ms": 174.3344013695605, "ttft_relative": 1.300333, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7785.0852443137765, "interactivity_tok_s": 136.28959311590967, "output_per_gpu": 64.74923095287097, "total_per_gpu": null, "tpot_error_pct": 0.06, "tpot_ms": 7.337317377927263, "tpot_relative": 1.251571, "ttft_error_pct": 33.96, "ttft_ms": 279.00956669418645, "ttft_relative": 2.081089, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7080.340037375005, "interactivity_tok_s": 132.4848488646262, "output_per_gpu": 63.45868132407279, "total_per_gpu": 570.4556992114143, "tpot_error_pct": 2.93, "tpot_ms": 7.548032915233996, "tpot_relative": 1.287514, "ttft_error_pct": 31.57, "ttft_ms": 142.52038097500025, "ttft_relative": 1.063037, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 6948.579316699761, "interactivity_tok_s": 136.37241651219028, "output_per_gpu": 64.5952488682299, "total_per_gpu": 580.6727636000056, "tpot_ms": 7.332861186855997, "tpot_relative": 1.25081, "ttft_ms": 208.27603309589904, "ttft_relative": 1.553499, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9392.41456482956, "interactivity_tok_s": 112.5917134834999, "output_per_gpu": 107.94767039610288, "total_per_gpu": null, "tpot_error_pct": 1.42, "tpot_ms": 8.881648294184172, "tpot_relative": 1.514996, "ttft_error_pct": 19.97, "ttft_ms": 306.4883598791514, "ttft_relative": 2.286049, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8558.572247350017, "interactivity_tok_s": 110.37537101579332, "output_per_gpu": 106.46322081475611, "total_per_gpu": 947.0395712644275, "tpot_error_pct": 3.45, "tpot_ms": 9.05999219569475, "tpot_relative": 1.545418, "ttft_error_pct": 33.41, "ttft_ms": 170.11208851250004, "ttft_relative": 1.26884, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 8362.492824319634, "interactivity_tok_s": 114.18569482042417, "output_per_gpu": 108.84735154297385, "total_per_gpu": 968.2475163689469, "tpot_ms": 8.7576644480087, "tpot_relative": 1.493848, "ttft_ms": 255.46477265743306, "ttft_relative": 1.905472, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11394.28512659717, "interactivity_tok_s": 92.72802493788501, "output_per_gpu": 179.15104169790104, "total_per_gpu": null, "tpot_error_pct": 3.71, "tpot_ms": 10.78422624303561, "tpot_relative": 1.839531, "ttft_error_pct": 20.94, "ttft_ms": 362.02167997174047, "ttft_relative": 2.700264, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10223.343610099653, "interactivity_tok_s": 91.48211994699297, "output_per_gpu": 173.71073150531384, "total_per_gpu": 1567.6958725468771, "tpot_error_pct": 2.4, "tpot_ms": 10.931097798995312, "tpot_relative": 1.864583, "ttft_error_pct": 51.53, "ttft_ms": 221.93928453750004, "ttft_relative": 1.655411, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26306422380", "measured": {"e2e_ms": 10706.369251238357, "interactivity_tok_s": 89.28643004319073, "output_per_gpu": 165.9654861366776, "total_per_gpu": 1497.7969717072222, "tpot_ms": 11.199910216101905, "tpot_relative": 1.910436, "ttft_ms": 457.9034642967599, "ttft_relative": 3.415431, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 39}]}, {"aic": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.96, "tpot_shape_error_pct": 11.85, "ttft_mape_pct": 50.26, "ttft_shape_error_pct": 28.12}, "aisimulate": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 55.04, "tpot_shape_error_pct": 17.96, "ttft_mape_pct": 49.49, "ttft_shape_error_pct": 13.65}, "frameworks": ["trt"], "gpu_skus": ["b200"], "hf_model_paths": ["nvidia/DeepSeek-R1-0528-FP4-v2"], "model": "nvidia/DeepSeek-R1-0528-FP4-v2", "precisions": ["fp4"], "rows": 21, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.82, "tpot_shape_error_pct": 4.92, "ttft_mape_pct": 57.98, "ttft_shape_error_pct": 35.98}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.23, "tpot_shape_error_pct": 3.08, "ttft_mape_pct": 49.12, "ttft_shape_error_pct": 16.32}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.82, "tpot_shape_error_pct": 4.92, "ttft_mape_pct": 57.98, "ttft_shape_error_pct": 35.98}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.23, "tpot_shape_error_pct": 3.08, "ttft_mape_pct": 49.12, "ttft_shape_error_pct": 16.32}, "gpu": "b200", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.6, "tpot_shape_error_pct": null, "ttft_mape_pct": 75.23, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 62.0, "tpot_shape_error_pct": null, "ttft_mape_pct": 79.03, "ttft_shape_error_pct": null}, "framework": "trt", "id": "d911ecbe303791f9", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 30885.37204863349, "interactivity_tok_s": 33.49395496528785, "output_per_gpu": 1063.5139071845126, "total_per_gpu": null, "tpot_error_pct": 8.6, "tpot_ms": 29.856133772090235, "tpot_relative": 1.086031, "ttft_error_pct": 75.23, "ttft_ms": 342.547199785178, "ttft_relative": 0.247731, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41386.02923769895, "interactivity_tok_s": 22.45446823576536, "output_per_gpu": 699.1923009929268, "total_per_gpu": 1397.7088348948078, "tpot_error_pct": 62.0, "tpot_ms": 44.53456610507503, "tpot_relative": 1.619966, "ttft_error_pct": 79.03, "ttft_ms": 289.92408529375035, "ttft_relative": 0.209674, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 26711.515620773298, "interactivity_tok_s": 36.375472848148625, "output_per_gpu": 1060.3329784977325, "total_per_gpu": 2119.641148610988, "tpot_ms": 27.49105157133088, "tpot_relative": 1.0, "ttft_ms": 1382.7394111172907, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.53, "tpot_shape_error_pct": 6.33, "ttft_mape_pct": 51.61, "ttft_shape_error_pct": 39.74}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.37, "tpot_shape_error_pct": 3.43, "ttft_mape_pct": 48.03, "ttft_shape_error_pct": 10.46}, "framework": "trt", "id": "40940c2161a8e4e7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10272.74165167395, "interactivity_tok_s": 100.6474364796678, "output_per_gpu": 199.27522878266396, "total_per_gpu": null, "tpot_error_pct": 16.57, "tpot_ms": 9.935672829600723, "tpot_relative": 0.834315, "ttft_error_pct": 31.06, "ttft_ms": 108.54834699241053, "ttft_relative": 0.689407, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9324.902105762474, "interactivity_tok_s": 99.0111113107711, "output_per_gpu": 192.28746988617294, "total_per_gpu": 386.62190518879316, "tpot_error_pct": 15.19, "tpot_ms": 10.099876536697485, "tpot_relative": 0.848103, "ttft_error_pct": 47.8, "ttft_ms": 82.19604181874926, "ttft_relative": 0.52204, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 11055.383215170878, "interactivity_tok_s": 83.9716277824464, "output_per_gpu": 162.21749620531003, "total_per_gpu": 326.1618527456772, "tpot_ms": 11.908784269262933, "tpot_relative": 1.0, "ttft_ms": 157.45167700952152, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11644.444730625692, "interactivity_tok_s": 88.94064923086972, "output_per_gpu": 352.655715128822, "total_per_gpu": null, "tpot_error_pct": 19.45, "tpot_ms": 11.243452894122992, "tpot_relative": 0.944131, "ttft_error_pct": 37.55, "ttft_ms": 142.39241993787132, "ttft_relative": 0.904356, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10784.245139312623, "interactivity_tok_s": 86.57269483232815, "output_per_gpu": 334.0684883918848, "total_per_gpu": 667.0802307931734, "tpot_error_pct": 17.25, "tpot_ms": 11.5509861618236, "tpot_relative": 0.969955, "ttft_error_pct": 46.18, "ttft_ms": 122.71659720312527, "ttft_relative": 0.779392, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 13112.7251516853, "interactivity_tok_s": 71.63910509834686, "output_per_gpu": 274.9217339574204, "total_per_gpu": 548.9738185759111, "tpot_ms": 13.958856669512974, "tpot_relative": 1.172148, "ttft_ms": 228.0174952262314, "ttft_relative": 1.448174, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15079.760285118264, "interactivity_tok_s": 68.60278427751268, "output_per_gpu": 545.2855635960007, "total_per_gpu": null, "tpot_error_pct": 22.66, "tpot_ms": 14.5766678500218, "tpot_relative": 1.224027, "ttft_error_pct": 64.54, "ttft_ms": 167.82907454596338, "ttft_relative": 1.065908, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14246.555074027856, "interactivity_tok_s": 65.63896266971591, "output_per_gpu": 504.02418505583813, "total_per_gpu": 1008.2756231559026, "tpot_error_pct": 19.17, "tpot_ms": 15.234853802181942, "tpot_relative": 1.279295, "ttft_error_pct": 56.45, "ttft_ms": 206.13962696875018, "ttft_relative": 1.309225, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 17851.475640733042, "interactivity_tok_s": 53.05415636335744, "output_per_gpu": 402.98366788167647, "total_per_gpu": 806.1490319359076, "tpot_ms": 18.84866462018918, "tpot_relative": 1.582753, "ttft_ms": 473.3082886221382, "ttft_relative": 3.006054, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19553.6691821419, "interactivity_tok_s": 52.78618338440121, "output_per_gpu": 840.5751706991762, "total_per_gpu": null, "tpot_error_pct": 23.44, "tpot_ms": 18.944351265515227, "tpot_relative": 1.590788, "ttft_error_pct": 73.27, "ttft_ms": 173.59783751982354, "ttft_relative": 1.102547, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19072.916658336828, "interactivity_tok_s": 49.201802370105995, "output_per_gpu": 752.9843843056407, "total_per_gpu": 1507.6229541989285, "tpot_error_pct": 17.86, "tpot_ms": 20.324458695187545, "tpot_relative": 1.706678, "ttft_error_pct": 41.69, "ttft_ms": 378.72955550546857, "ttft_relative": 2.40537, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 23410.442915636122, "interactivity_tok_s": 40.411998000590465, "output_per_gpu": 614.2069847097996, "total_per_gpu": 1229.7632833803077, "tpot_ms": 24.745126434614516, "tpot_relative": 2.077889, "ttft_ms": 649.4595694752206, "ttft_relative": 4.124818, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.55, "tpot_shape_error_pct": null, "ttft_mape_pct": 67.86, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 58.19, "tpot_shape_error_pct": null, "ttft_mape_pct": 60.29, "ttft_shape_error_pct": null}, "framework": "trt", "id": "d9719c0607bb2a7e", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 43502.31001022957, "interactivity_tok_s": 23.72577591871908, "output_per_gpu": 1511.2927430102836, "total_per_gpu": null, "tpot_error_pct": 21.55, "tpot_ms": 42.14825274527791, "tpot_relative": 1.215497, "ttft_error_pct": 67.86, "ttft_ms": 384.64745181026933, "ttft_relative": 0.321414, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51066.931705168754, "interactivity_tok_s": 18.230528427793313, "output_per_gpu": 1133.5213417431796, "total_per_gpu": 2265.9471387861954, "tpot_error_pct": 58.19, "tpot_ms": 54.853045207151105, "tpot_relative": 1.581885, "ttft_error_pct": 60.29, "ttft_ms": 475.1834076574236, "ttft_relative": 0.397067, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 33148.52787621712, "interactivity_tok_s": 28.8386075574554, "output_per_gpu": 1711.2950278471235, "total_per_gpu": 3420.9360946005, "tpot_ms": 34.67573800183284, "tpot_relative": 1.0, "ttft_ms": 1196.7344347315702, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.15, "tpot_shape_error_pct": 0.68, "ttft_mape_pct": 57.15, "ttft_shape_error_pct": 24.69}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.09, "tpot_shape_error_pct": 2.05, "ttft_mape_pct": 30.76, "ttft_shape_error_pct": 33.92}, "framework": "trt", "id": "4e28dc1af7010d51", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 20552.788879239433, "interactivity_tok_s": 50.32372007857289, "output_per_gpu": 800.1194533679021, "total_per_gpu": null, "tpot_error_pct": 10.84, "tpot_ms": 19.871344933137912, "tpot_relative": 0.891554, "ttft_error_pct": 51.12, "ttft_ms": 224.40301263934958, "ttft_relative": 0.488798, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19365.092538890633, "interactivity_tok_s": 48.26645284497119, "output_per_gpu": 744.7496520603495, "total_per_gpu": 1489.8350948042323, "tpot_error_pct": 7.04, "tpot_ms": 20.718323826528895, "tpot_relative": 0.929555, "ttft_error_pct": 40.8, "ttft_ms": 271.79510548437463, "ttft_relative": 0.592028, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 21004.653071637586, "interactivity_tok_s": 44.86632560582345, "output_per_gpu": 684.880678028515, "total_per_gpu": 1370.0701531815105, "tpot_ms": 22.288430944525675, "tpot_relative": 1.0, "ttft_ms": 459.09146991143643, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26174.084316354725, "interactivity_tok_s": 39.43444452507828, "output_per_gpu": 1255.922954885928, "total_per_gpu": null, "tpot_error_pct": 11.45, "tpot_ms": 25.358541550244265, "tpot_relative": 1.137745, "ttft_error_pct": 63.19, "ttft_ms": 232.2963104548432, "ttft_relative": 0.505991, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25486.983405585586, "interactivity_tok_s": 36.810470403026436, "output_per_gpu": 1128.9173900098951, "total_per_gpu": 2260.314829958569, "tpot_error_pct": 5.13, "tpot_ms": 27.166183671420384, "tpot_relative": 1.218847, "ttft_error_pct": 20.71, "ttft_ms": 500.32266709765446, "ttft_relative": 1.08981, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 26963.414251562426, "interactivity_tok_s": 34.92045940975033, "output_per_gpu": 1064.6747528481374, "total_per_gpu": 2131.6884248935394, "tpot_ms": 28.636507563264892, "tpot_relative": 1.284815, "ttft_ms": 631.0395196405807, "ttft_relative": 1.37454, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 8}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.48, "tpot_shape_error_pct": 4.85, "ttft_mape_pct": 59.97, "ttft_shape_error_pct": 41.42}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.27, "tpot_shape_error_pct": 4.52, "ttft_mape_pct": 49.54, "ttft_shape_error_pct": 9.41}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.48, "tpot_shape_error_pct": 4.85, "ttft_mape_pct": 59.97, "ttft_shape_error_pct": 41.42}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.27, "tpot_shape_error_pct": 4.52, "ttft_mape_pct": 49.54, "ttft_shape_error_pct": 9.41}, "gpu": "b200", "precisions": ["fp4"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 81.45, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.42, "ttft_shape_error_pct": null}, "framework": "trt", "id": "72866619a0285262", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 237045.76705372066, "interactivity_tok_s": 34.60491371387326, "output_per_gpu": 1106.224160724237, "total_per_gpu": null, "tpot_error_pct": 9.56, "tpot_ms": 28.89763021137359, "tpot_relative": 1.095637, "ttft_error_pct": 81.45, "ttft_ms": 345.277992359602, "ttft_relative": 0.185526, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 229869.5692178541, "interactivity_tok_s": 32.140344913572434, "output_per_gpu": 995.7591391905888, "total_per_gpu": 1120.0811157984408, "tpot_error_pct": 17.97, "tpot_ms": 31.113542891000947, "tpot_relative": 1.179652, "ttft_error_pct": 84.42, "ttft_ms": 289.90807434531285, "ttft_relative": 0.155775, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 196423.44211424462, "interactivity_tok_s": 37.91441520654352, "output_per_gpu": 1152.9867404267757, "total_per_gpu": 1296.9388116967593, "tpot_ms": 26.375192510615687, "tpot_relative": 1.0, "ttft_ms": 1861.075287787935, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.14, "tpot_shape_error_pct": 4.12, "ttft_mape_pct": 51.38, "ttft_shape_error_pct": 38.44}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.07, "tpot_shape_error_pct": 3.48, "ttft_mape_pct": 41.76, "ttft_shape_error_pct": 9.29}, "framework": "trt", "id": "6058cc9ed2841831", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 90538.29619954295, "interactivity_tok_s": 90.61438326832732, "output_per_gpu": 362.0435169976232, "total_per_gpu": null, "tpot_error_pct": 19.36, "tpot_ms": 11.03577560130603, "tpot_relative": 0.806421, "ttft_error_pct": 34.63, "ttft_ms": 144.25824924526276, "ttft_relative": 0.653682, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 81377.34585889679, "interactivity_tok_s": 91.25797331642049, "output_per_gpu": 355.4129496086755, "total_per_gpu": 399.5411020425051, "tpot_error_pct": 19.93, "tpot_ms": 10.957946617252622, "tpot_relative": 0.800734, "ttft_error_pct": 45.16, "ttft_ms": 121.02344027187473, "ttft_relative": 0.548398, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 101703.30407098227, "interactivity_tok_s": 73.0733634446586, "output_per_gpu": 284.4120391240986, "total_per_gpu": 319.7247024086054, "tpot_ms": 13.684877127044253, "tpot_relative": 1.0, "ttft_ms": 220.6855685406481, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 120209.9952818675, "interactivity_tok_s": 68.23792340005879, "output_per_gpu": 545.4432240029563, "total_per_gpu": null, "tpot_error_pct": 13.48, "tpot_ms": 14.654607733844646, "tpot_relative": 1.070861, "ttft_error_pct": 50.04, "ttft_ms": 174.10333394599726, "ttft_relative": 0.78892, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 106956.6935158364, "interactivity_tok_s": 69.18410565364235, "output_per_gpu": 537.1403710774227, "total_per_gpu": 604.2072122115688, "tpot_error_pct": 14.66, "tpot_ms": 14.454186992115188, "tpot_relative": 1.056216, "ttft_error_pct": 41.64, "ttft_ms": 203.39334772187584, "ttft_relative": 0.921643, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 125446.21063051854, "interactivity_tok_s": 59.03998723022972, "output_per_gpu": 457.4504191756692, "total_per_gpu": 514.5672479257856, "tpot_ms": 16.937673040145558, "tpot_relative": 1.237693, "ttft_ms": 348.5161443859397, "ttft_relative": 1.579243, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 153054.14948313942, "interactivity_tok_s": 53.58154487086771, "output_per_gpu": 856.7390323594582, "total_per_gpu": null, "tpot_error_pct": 18.59, "tpot_ms": 18.663142363849612, "tpot_relative": 1.363779, "ttft_error_pct": 69.47, "ttft_ms": 184.3503808472599, "ttft_relative": 0.835353, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 136135.44821998556, "interactivity_tok_s": 54.26101472297765, "output_per_gpu": 843.0154675275109, "total_per_gpu": 948.5326471715284, "tpot_error_pct": 19.61, "tpot_ms": 18.429437877366766, "tpot_relative": 1.346701, "ttft_error_pct": 38.49, "ttft_ms": 371.47104754609364, "ttft_relative": 1.683259, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 169491.1945921436, "interactivity_tok_s": 43.61875858622759, "output_per_gpu": 677.8621081521316, "total_per_gpu": 762.7076425401787, "tpot_ms": 22.9259161060981, "tpot_relative": 1.675274, "ttft_ms": 603.8783825541032, "ttft_relative": 2.736375, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.29, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.91, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.98, "tpot_shape_error_pct": null, "ttft_mape_pct": 81.65, "ttft_shape_error_pct": null}, "framework": "trt", "id": "dc3241fdab74a890", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 334985.4832664996, "interactivity_tok_s": 24.480405846470113, "output_per_gpu": 1565.7547492872488, "total_per_gpu": null, "tpot_error_pct": 21.29, "tpot_ms": 40.84899597954142, "tpot_relative": 1.212946, "ttft_error_pct": 84.91, "ttft_ms": 391.35719807581347, "ttft_relative": 0.150943, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 315965.9953505331, "interactivity_tok_s": 23.38524776022817, "output_per_gpu": 1455.2112426085753, "total_per_gpu": 1636.8964824850825, "tpot_error_pct": 26.98, "tpot_ms": 42.762001508520385, "tpot_relative": 1.26975, "ttft_error_pct": 81.65, "ttft_ms": 475.74334446211, "ttft_relative": 0.183491, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 251030.55389819603, "interactivity_tok_s": 29.693421714119783, "output_per_gpu": 1812.8166205570622, "total_per_gpu": 2039.1494119170457, "tpot_ms": 33.677492935227505, "tpot_relative": 1.0, "ttft_ms": 2592.739917309973, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.05, "tpot_shape_error_pct": 6.3, "ttft_mape_pct": 49.65, "ttft_shape_error_pct": 47.37}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.37, "tpot_shape_error_pct": 6.61, "ttft_mape_pct": 27.7, "ttft_shape_error_pct": 9.64}, "framework": "trt", "id": "af4644cd29d3128a", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 164197.83301333222, "interactivity_tok_s": 49.95580431092467, "output_per_gpu": 798.6370612226245, "total_per_gpu": null, "tpot_error_pct": 10.22, "tpot_ms": 20.017693915525914, "tpot_relative": 0.897818, "ttft_error_pct": 34.02, "ttft_ms": 232.90215125943888, "ttft_relative": 0.659776, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 146084.2857798517, "interactivity_tok_s": 50.64974099646911, "output_per_gpu": 789.1594373166977, "total_per_gpu": 887.6931418413988, "tpot_error_pct": 11.45, "tpot_ms": 19.743437583811374, "tpot_relative": 0.885517, "ttft_error_pct": 24.03, "ttft_ms": 268.15815195156034, "ttft_relative": 0.759651, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 165022.8355744188, "interactivity_tok_s": 44.851226692243955, "output_per_gpu": 696.431181186952, "total_per_gpu": 783.3869229343893, "tpot_ms": 22.295934219630347, "tpot_relative": 1.0, "ttft_ms": 353.0016616292414, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 208375.95550911786, "interactivity_tok_s": 39.35566603536338, "output_per_gpu": 1258.5622126773249, "total_per_gpu": null, "tpot_error_pct": 15.87, "tpot_ms": 25.409301905891805, "tpot_relative": 1.139638, "ttft_error_pct": 65.28, "ttft_ms": 248.3635979580826, "ttft_relative": 0.703576, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 184502.0544926796, "interactivity_tok_s": 40.033866241016085, "output_per_gpu": 1246.1796783745049, "total_per_gpu": 1402.1594558006832, "tpot_error_pct": 17.3, "tpot_ms": 24.97885150486578, "tpot_relative": 1.120332, "ttft_error_pct": 31.36, "ttft_ms": 490.97374278593963, "ttft_relative": 1.390854, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 223222.52817488933, "interactivity_tok_s": 33.108536855840725, "output_per_gpu": 1030.0892079695977, "total_per_gpu": 1159.0217272334212, "tpot_ms": 30.203690496929603, "tpot_relative": 1.354673, "ttft_ms": 715.2788263669208, "ttft_relative": 2.026276, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:8192", "label": "1k8k", "precisions": ["fp4"], "rows": 7}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.35, "tpot_shape_error_pct": 24.04, "ttft_mape_pct": 28.65, "ttft_shape_error_pct": 10.28}, "aisimulate": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 137.7, "tpot_shape_error_pct": 42.92, "ttft_mape_pct": 49.93, "ttft_shape_error_pct": 14.17}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.35, "tpot_shape_error_pct": 24.04, "ttft_mape_pct": 28.65, "ttft_shape_error_pct": 10.28}, "aisimulate": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 137.7, "tpot_shape_error_pct": 42.92, "ttft_mape_pct": 49.93, "ttft_shape_error_pct": 14.17}, "gpu": "b200", "precisions": ["fp4"], "rows": 6, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.26, "tpot_shape_error_pct": 27.05, "ttft_mape_pct": 27.27, "ttft_shape_error_pct": 9.78}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 185.52, "tpot_shape_error_pct": 49.88, "ttft_mape_pct": 53.77, "ttft_shape_error_pct": 7.66}, "framework": "trt", "id": "9abd0f1fa3ad7aea", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 21864.408333683456, "interactivity_tok_s": 50.681499216098814, "output_per_gpu": 365.6054142712327, "total_per_gpu": null, "tpot_error_pct": 23.06, "tpot_ms": 19.731065881380896, "tpot_relative": 0.76935, "ttft_error_pct": 22.2, "ttft_ms": 1679.5279370307992, "ttft_relative": 0.777995, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51687.73316976404, "interactivity_tok_s": 18.19789658684404, "output_per_gpu": 141.167571371667, "total_per_gpu": 1272.4531457439134, "tpot_error_pct": 114.27, "tpot_ms": 54.95140579724684, "tpot_relative": 2.142655, "ttft_error_pct": 56.02, "ttft_ms": 949.5249453671893, "ttft_relative": 0.439841, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 25777.137014257823, "interactivity_tok_s": 38.991813058218234, "output_per_gpu": 277.5582591795923, "total_per_gpu": 2501.848523627442, "tpot_ms": 25.646409375909535, "tpot_relative": 1.0, "ttft_ms": 2158.7906952470803, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29269.37552559236, "interactivity_tok_s": 37.487897071858, "output_per_gpu": 555.2689075990302, "total_per_gpu": null, "tpot_error_pct": 16.61, "tpot_ms": 26.67527597195351, "tpot_relative": 1.040117, "ttft_error_pct": 27.84, "ttft_ms": 1980.5682062839176, "ttft_relative": 0.917443, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 86318.65675207874, "interactivity_tok_s": 10.838507884764942, "output_per_gpu": 169.29613707594694, "total_per_gpu": 1528.4193815923177, "tpot_error_pct": 188.42, "tpot_ms": 92.26362250523816, "tpot_relative": 3.597526, "ttft_error_pct": 54.41, "ttft_ms": 1251.4388058164054, "ttft_relative": 0.579694, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 32179.55576365803, "interactivity_tok_s": 31.25993831961658, "output_per_gpu": 444.6297438303651, "total_per_gpu": 4014.1537180963232, "tpot_ms": 31.989826396185464, "tpot_relative": 1.247341, "ttft_ms": 2744.8580625788964, "ttft_relative": 1.271479, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 53748.00219750394, "interactivity_tok_s": 19.995676429095035, "output_per_gpu": 614.0622814842503, "total_per_gpu": null, "tpot_error_pct": 12.11, "tpot_ms": 50.01081126442583, "tpot_relative": 1.950012, "ttft_error_pct": 31.77, "ttft_ms": 2586.9422739963115, "ttft_relative": 1.198329, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 147805.25955955658, "interactivity_tok_s": 6.3345704225837345, "output_per_gpu": 198.60918590955683, "total_per_gpu": 1786.9687445487511, "tpot_error_pct": 253.88, "tpot_ms": 157.86390130494777, "tpot_relative": 6.1554, "ttft_error_pct": 50.88, "ttft_ms": 1862.3110237257729, "ttft_relative": 0.862664, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 44967.357353454194, "interactivity_tok_s": 22.416526016835636, "output_per_gpu": 641.1767104386828, "total_per_gpu": 5768.931260854529, "tpot_ms": 44.60994532555862, "tpot_relative": 1.739423, "ttft_ms": 3791.453647895014, "ttft_relative": 1.756286, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.45, "tpot_shape_error_pct": 21.03, "ttft_mape_pct": 30.03, "ttft_shape_error_pct": 10.79}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 89.88, "tpot_shape_error_pct": 35.95, "ttft_mape_pct": 46.09, "ttft_shape_error_pct": 20.67}, "framework": "trt", "id": "89a8867e4c1e1c08", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 30124.29413939497, "interactivity_tok_s": 35.750768799957974, "output_per_gpu": 540.7740870319969, "total_per_gpu": null, "tpot_error_pct": 14.53, "tpot_ms": 27.97142644946912, "tpot_relative": 0.854717, "ttft_error_pct": 24.61, "ttft_ms": 1509.5248815880573, "ttft_relative": 0.753912, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 47205.934969879876, "interactivity_tok_s": 19.949971760910994, "output_per_gpu": 308.47698882708613, "total_per_gpu": 2780.5430879674173, "tpot_error_pct": 53.17, "tpot_ms": 50.12538423534772, "tpot_relative": 1.531671, "ttft_error_pct": 52.62, "ttft_ms": 948.6363031781245, "ttft_relative": 0.473784, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 32156.403091490574, "interactivity_tok_s": 30.556802552042612, "output_per_gpu": 445.50254875849225, "total_per_gpu": 4015.6610622151024, "tpot_ms": 32.72593715578902, "tpot_relative": 1.0, "ttft_ms": 2002.254807645295, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 42694.07693226752, "interactivity_tok_s": 25.12181183990034, "output_per_gpu": 772.8559230546763, "total_per_gpu": null, "tpot_error_pct": 9.46, "tpot_ms": 39.806046091457674, "tpot_relative": 1.216345, "ttft_error_pct": 27.04, "ttft_ms": 1972.4917807063175, "ttft_relative": 0.985135, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 80573.20938432976, "interactivity_tok_s": 11.639835852806039, "output_per_gpu": 362.19685179285483, "total_per_gpu": 3269.942822047838, "tpot_error_pct": 95.41, "tpot_ms": 85.91186444943963, "tpot_relative": 2.625192, "ttft_error_pct": 48.12, "ttft_ms": 1402.5844525281213, "ttft_relative": 0.700502, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 43161.893853526635, "interactivity_tok_s": 22.7457681422515, "output_per_gpu": 663.4387193158126, "total_per_gpu": 5989.5790572361575, "tpot_ms": 43.96422199268117, "tpot_relative": 1.343406, "ttft_ms": 2703.4283865710677, "ttft_relative": 1.350192, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 77712.32444715733, "interactivity_tok_s": 13.565438893975177, "output_per_gpu": 850.3976201671476, "total_per_gpu": null, "tpot_error_pct": 16.36, "tpot_ms": 73.71674501767357, "tpot_relative": 2.252548, "ttft_error_pct": 38.45, "ttft_ms": 2300.0942940772625, "ttft_relative": 1.148752, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 131721.19058841275, "interactivity_tok_s": 7.140429129273054, "output_per_gpu": 444.8307997450043, "total_per_gpu": 4002.326136712178, "tpot_error_pct": 121.06, "tpot_ms": 140.04760524831468, "tpot_relative": 4.279407, "ttft_error_pct": 37.54, "ttft_ms": 2333.907613694928, "ttft_relative": 1.16564, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 62220.76996093574, "interactivity_tok_s": 15.784495220378735, "output_per_gpu": 928.5447313390914, "total_per_gpu": 8354.499844604272, "tpot_ms": 63.35330880324508, "tpot_relative": 1.935875, "ttft_ms": 3736.682455434675, "ttft_relative": 1.866237, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 6}]}, {"aic": {"coverage_pct": 15.38, "points": 4, "status_counts": {"failed": 22, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 54.63, "tpot_shape_error_pct": 7.18, "ttft_mape_pct": 96.28, "ttft_shape_error_pct": 49.37}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 26, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "frameworks": ["dynamo-trt"], "gpu_skus": ["gb200"], "hf_model_paths": ["nvidia/DeepSeek-R1-0528-NVFP4-v2"], "model": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "precisions": ["fp4"], "rows": 26, "workloads": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp4"], "rows": 11, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "d31d1c91c5689ea0", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 26972.177067096392, "interactivity_tok_s": 38.179070317525, "output_per_gpu": 8828.108470526216, "total_per_gpu": 11774.886850647446, "tpot_ms": 26.192361199035766, "tpot_relative": 1.0, "ttft_ms": 2865.0872098096947, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "98d590bad39e3f89", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 6144, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 60758.9085106188, "interactivity_tok_s": 18.57017717699265, "output_per_gpu": 11250.737196360149, "total_per_gpu": 15003.033787381684, "tpot_ms": 53.84978239404957, "tpot_relative": 1.0, "ttft_ms": 11188.083111310394, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "62695264f54eea20", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 8992.148507712409, "interactivity_tok_s": 175.06565084998354, "output_per_gpu": 15.582542854480305, "total_per_gpu": 27.75896956005298, "tpot_ms": 5.712142816964794, "tpot_relative": 1.0, "ttft_ms": 3720.886858254671, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 6567.758506323056, "interactivity_tok_s": 143.99557542755574, "output_per_gpu": 50.603039834353936, "total_per_gpu": 90.42842522043986, "tpot_ms": 6.944657827372624, "tpot_relative": 1.215771, "ttft_ms": 208.55319916348284, "ttft_relative": 0.056049, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 7068.093570979545, "interactivity_tok_s": 133.90362583990225, "output_per_gpu": 94.89155859064786, "total_per_gpu": 169.07090612277543, "tpot_ms": 7.468057670041133, "tpot_relative": 1.3074, "ttft_ms": 220.9750996592144, "ttft_relative": 0.059388, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 48, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 8213.558379573322, "interactivity_tok_s": 116.40117577452594, "output_per_gpu": 162.17324442471755, "total_per_gpu": 287.3404896751843, "tpot_ms": 8.59097851328446, "tpot_relative": 1.503985, "ttft_ms": 266.4142356584004, "ttft_relative": 0.0716, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 96, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 9570.589786951314, "interactivity_tok_s": 99.35737821239346, "output_per_gpu": 280.1847769540075, "total_per_gpu": 498.6656426255856, "tpot_ms": 10.064677812475368, "tpot_relative": 1.761979, "ttft_ms": 311.1405712479609, "ttft_relative": 0.08362, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 14573.700193033195, "interactivity_tok_s": 93.9997086498466, "output_per_gpu": 363.108829476633, "total_per_gpu": 646.027644354837, "tpot_ms": 10.638330845524722, "tpot_relative": 1.862406, "ttft_ms": 4789.532199038998, "ttft_relative": 1.287202, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "724e7c319c1ae2dd", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 15302.787980297371, "interactivity_tok_s": 79.1419340348775, "output_per_gpu": 1188.9521771932375, "total_per_gpu": 2113.8348071949786, "tpot_ms": 12.635526439868206, "tpot_relative": 1.0, "ttft_ms": 3670.2276964869084, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "49ec51f75ad6c7a3", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 44150.14754770982, "interactivity_tok_s": 23.455536220088856, "output_per_gpu": 10851.981312348567, "total_per_gpu": 14474.320865179105, "tpot_ms": 42.63385797778243, "tpot_relative": 1.0, "ttft_ms": 4909.563350885137, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "92026887691cde0e", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 15950.917813692473, "interactivity_tok_s": 66.37825073392355, "output_per_gpu": 3876.5706157303457, "total_per_gpu": 6200.1293376011245, "tpot_ms": 15.065175549872329, "tpot_relative": 1.0, "ttft_ms": 2068.2023482587556, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 11}, {"aic": {"coverage_pct": 57.14, "points": 4, "status_counts": {"failed": 3, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 54.63, "tpot_shape_error_pct": 7.18, "ttft_mape_pct": 96.28, "ttft_shape_error_pct": 49.37}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 57.14, "points": 4, "status_counts": {"failed": 3, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 54.63, "tpot_shape_error_pct": 7.18, "ttft_mape_pct": 96.28, "ttft_shape_error_pct": 49.37}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp4"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 50.0, "points": 2, "status_counts": {"failed": 2, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 90.84, "tpot_shape_error_pct": 9.38, "ttft_mape_pct": 94.08, "ttft_shape_error_pct": 72.18}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "606ae6c65eb044d7", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 563, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 106953.99724042248, "interactivity_tok_s": 75.55685171967497, "output_per_gpu": 1149.9211964023195, "total_per_gpu": 1150.0324811495238, "tpot_ms": 13.235067068571366, "tpot_relative": 1.0, "ttft_ms": 9474.41749747927, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 286135.672, "interactivity_tok_s": 28.639344731792534, "output_per_gpu": 1499.1633055555556, "total_per_gpu": null, "tpot_error_pct": 100.23, "tpot_ms": 34.917, "tpot_relative": 2.638219, "ttft_error_pct": 90.74, "ttft_ms": 130.525, "ttft_relative": 0.013777, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 129984.48071985257, "interactivity_tok_s": 57.3438746276959, "output_per_gpu": 3460.8512050721583, "total_per_gpu": 3460.9259885779675, "tpot_ms": 17.4386541978979, "tpot_relative": 1.31761, "ttft_ms": 1409.219117241729, "ttft_relative": 0.148739, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 350819.999, "interactivity_tok_s": 23.356845891530806, "output_per_gpu": 2445.2209722222224, "total_per_gpu": null, "tpot_error_pct": 81.45, "tpot_ms": 42.814, "tpot_relative": 3.234891, "ttft_error_pct": 97.42, "ttft_ms": 130.525, "ttft_relative": 0.013777, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 179036.97600766525, "interactivity_tok_s": 42.38089189885151, "output_per_gpu": 5082.619005352276, "total_per_gpu": 5082.822966819277, "tpot_ms": 23.595539291307347, "tpot_relative": 1.782805, "ttft_ms": 5066.100083601255, "ttft_relative": 0.534714, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 288902.94085290766, "interactivity_tok_s": 29.096219926123126, "output_per_gpu": 6365.7719218703, "total_per_gpu": 6365.803124198855, "tpot_ms": 34.36872564680409, "tpot_relative": 2.596793, "ttft_ms": 35515.671130932395, "ttft_relative": 3.748586, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.41, "tpot_shape_error_pct": 4.98, "ttft_mape_pct": 98.48, "ttft_shape_error_pct": 26.56}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "64f150aecdc194b5", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 73104.144, "interactivity_tok_s": 112.24604332697271, "output_per_gpu": 97.152, "total_per_gpu": null, "tpot_error_pct": 15.54, "tpot_ms": 8.909, "tpot_relative": 1.15536, "ttft_error_pct": 98.24, "ttft_ms": 130.525, "ttft_relative": 0.017567, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 64434.89786231115, "interactivity_tok_s": 129.68464412939773, "output_per_gpu": 108.05328653487096, "total_per_gpu": 113.92338710466292, "tpot_ms": 7.711013179033073, "tpot_relative": 1.0, "ttft_ms": 7429.978149069162, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 96898.999, "interactivity_tok_s": 84.64533604198408, "output_per_gpu": 164.275203125, "total_per_gpu": null, "tpot_error_pct": 21.29, "tpot_ms": 11.814, "tpot_relative": 1.532094, "ttft_error_pct": 98.71, "ttft_ms": 130.525, "ttft_relative": 0.017567, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 135, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 81839.44137034602, "interactivity_tok_s": 102.66895032908612, "output_per_gpu": 192.57141724036805, "total_per_gpu": 203.14207664789544, "tpot_ms": 9.740043087951003, "tpot_relative": 1.263134, "ttft_ms": 10117.517585265867, "ttft_relative": 1.361716, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "a4e6a284c2b3b00e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 7, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 68166.04443062762, "interactivity_tok_s": 170.2269015818259, "output_per_gpu": 12.975338130611902, "total_per_gpu": 13.631986048926596, "tpot_ms": 5.874512140604948, "tpot_relative": 1.0, "ttft_ms": 24967.908284014888, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 60}]}], "identity": "1024:8192", "label": "1k8k", "precisions": ["fp4"], "rows": 7}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "2bad0ee9b0e828ac", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 42038.72377699336, "interactivity_tok_s": 24.2641981837435, "output_per_gpu": 5434.1003726597955, "total_per_gpu": 13977.154179823427, "tpot_ms": 41.2129835252491, "tpot_relative": 1.0, "ttft_ms": 4108.8588127370485, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "94b32521b2b809b2", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 9139.618767565116, "interactivity_tok_s": 165.35035928508282, "output_per_gpu": 14.810623044423483, "total_per_gpu": 117.36894908689293, "tpot_ms": 6.047764300746914, "tpot_relative": 1.0, "ttft_ms": 3556.691938797012, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 7072.0772384200245, "interactivity_tok_s": 139.207086020689, "output_per_gpu": 46.68746920024747, "total_per_gpu": 374.75425024924994, "tpot_ms": 7.183542365446683, "tpot_relative": 1.187801, "ttft_ms": 503.4414727861682, "ttft_relative": 0.141548, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 44, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 8922.218804822347, "interactivity_tok_s": 113.88440214504487, "output_per_gpu": 137.50377426941668, "total_per_gpu": 1095.3889101004854, "tpot_ms": 8.780833732844162, "tpot_relative": 1.451914, "ttft_ms": 795.9752880936404, "ttft_relative": 0.223797, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 76, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 10940.268678843706, "interactivity_tok_s": 102.62590402769115, "output_per_gpu": 192.29690410534718, "total_per_gpu": 1544.4732259993664, "tpot_ms": 9.744128536301847, "tpot_relative": 1.611195, "ttft_ms": 1985.5829540640116, "ttft_relative": 0.558267, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "337dea5a8e2422d9", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 18261.863817112742, "interactivity_tok_s": 62.881488173230395, "output_per_gpu": 1851.2455108785407, "total_per_gpu": 8882.514699919386, "tpot_ms": 15.902931515315428, "tpot_relative": 1.0, "ttft_ms": 3614.795391518203, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 60}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "63fa398efdf14508", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 333, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 15732.248813422066, "interactivity_tok_s": 78.75270220113238, "output_per_gpu": 581.345305081689, "total_per_gpu": 4181.088388195699, "tpot_ms": 12.697976984282084, "tpot_relative": 1.0, "ttft_ms": 4030.4045312256235, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "a1f921238bcf7d73", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context MLA data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\" } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23438524239", "measured": {"e2e_ms": 28026.1337611173, "interactivity_tok_s": 37.98280353717481, "output_per_gpu": 4454.371372425083, "total_per_gpu": 13352.782270728028, "tpot_ms": 26.327703773137035, "tpot_relative": 1.0, "ttft_ms": 3767.4926978253, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 8}]}, {"aic": {"coverage_pct": 44.9, "points": 44, "status_counts": {"failed": 54, "success": 44, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.23, "tpot_shape_error_pct": 13.37, "ttft_mape_pct": 40.45, "ttft_shape_error_pct": 43.5}, "aisimulate": {"coverage_pct": 100.0, "points": 98, "status_counts": {"failed": 0, "success": 98, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.91, "tpot_shape_error_pct": 16.7, "ttft_mape_pct": 47.34, "ttft_shape_error_pct": 40.61}, "frameworks": ["dynamo-sglang", "dynamo-trt", "sglang"], "gpu_skus": ["b200", "b300", "gb200", "gb300"], "hf_model_paths": ["nvidia/GLM-5-NVFP4"], "model": "nvidia/GLM-5-NVFP4", "precisions": ["fp4"], "rows": 98, "workloads": [{"aic": {"coverage_pct": 41.18, "points": 21, "status_counts": {"failed": 30, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.72, "tpot_shape_error_pct": 15.41, "ttft_mape_pct": 33.0, "ttft_shape_error_pct": 29.91}, "aisimulate": {"coverage_pct": 100.0, "points": 51, "status_counts": {"failed": 0, "success": 51, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.41, "tpot_shape_error_pct": 17.86, "ttft_mape_pct": 52.54, "ttft_shape_error_pct": 40.06}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.75, "tpot_shape_error_pct": 13.83, "ttft_mape_pct": 19.59, "ttft_shape_error_pct": 22.47}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.96, "tpot_shape_error_pct": 14.04, "ttft_mape_pct": 60.27, "ttft_shape_error_pct": 9.88}, "gpu": "b200", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 21.63, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.3, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.98, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "896c307c67a408d6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9363.279692748278, "interactivity_tok_s": 110.95633908517412, "output_per_gpu": 54.347102217067814, "total_per_gpu": null, "tpot_error_pct": 16.68, "tpot_ms": 9.012554021202552, "tpot_relative": 0.833207, "ttft_error_pct": 21.63, "ttft_ms": 143.43692905806628, "ttft_relative": 0.783687, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8368.292539249993, "interactivity_tok_s": 110.45102858939464, "output_per_gpu": 53.132035241784735, "total_per_gpu": 106.78368149590435, "tpot_error_pct": 16.3, "tpot_ms": 9.053786214318865, "tpot_relative": 0.837019, "ttft_error_pct": 62.98, "ttft_ms": 67.74910185000039, "ttft_relative": 0.370156, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 10099.453910474404, "interactivity_tok_s": 92.4495578084875, "output_per_gpu": 43.97927030957737, "total_per_gpu": 88.38864108610123, "tpot_ms": 10.816709389476314, "tpot_relative": 1.0, "ttft_ms": 183.02839224979834, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.19, "tpot_shape_error_pct": 13.83, "ttft_mape_pct": 19.3, "ttft_shape_error_pct": 22.47}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.48, "tpot_shape_error_pct": 14.04, "ttft_mape_pct": 59.88, "ttft_shape_error_pct": 9.88}, "framework": "sglang", "id": "4a0deb805149456a", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11130.424694195382, "interactivity_tok_s": 93.28087267007932, "output_per_gpu": 91.46362924401097, "total_per_gpu": null, "tpot_error_pct": 9.31, "tpot_ms": 10.720311371194526, "tpot_relative": 0.906861, "ttft_error_pct": 9.0, "ttft_ms": 163.54616146338148, "ttft_relative": 0.910032, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9932.247169274997, "interactivity_tok_s": 93.01205117296485, "output_per_gpu": 89.75648057831953, "total_per_gpu": 180.3907452566586, "tpot_error_pct": 9.05, "tpot_ms": 10.751294992305931, "tpot_relative": 0.909482, "ttft_error_pct": 58.03, "ttft_ms": 75.42557767500055, "ttft_relative": 0.419696, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 11017.686353249701, "interactivity_tok_s": 84.59276478265762, "output_per_gpu": 80.72318502236429, "total_per_gpu": 162.2358119642314, "tpot_ms": 11.821341961919304, "tpot_relative": 1.0, "ttft_ms": 179.7147997242064, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14297.762259883444, "interactivity_tok_s": 72.46482076922645, "output_per_gpu": 142.7738852811683, "total_per_gpu": null, "tpot_error_pct": 2.92, "tpot_ms": 13.799799535620583, "tpot_relative": 1.167363, "ttft_error_pct": 6.61, "ttft_ms": 180.56733494358707, "ttft_relative": 1.004744, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12858.514866750033, "interactivity_tok_s": 72.66388625651649, "output_per_gpu": 141.8818936359252, "total_per_gpu": 282.7157938428199, "tpot_error_pct": 2.64, "tpot_ms": 13.761994458565312, "tpot_relative": 1.164165, "ttft_error_pct": 52.32, "ttft_ms": 80.75994020000061, "ttft_relative": 0.449378, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 12615.354953549831, "interactivity_tok_s": 74.58437464182121, "output_per_gpu": 144.020233407619, "total_per_gpu": 286.97667879838275, "tpot_ms": 13.40763403598046, "tpot_relative": 1.134189, "ttft_ms": 169.37167812484404, "ttft_relative": 0.942447, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17117.379561359317, "interactivity_tok_s": 60.521384497764814, "output_per_gpu": 239.12159888495339, "total_per_gpu": null, "tpot_error_pct": 2.82, "tpot_ms": 16.523085324277936, "tpot_relative": 1.397733, "ttft_error_pct": 0.76, "ttft_ms": 214.26327462298875, "ttft_relative": 1.192241, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15246.225467950037, "interactivity_tok_s": 60.38461077754694, "output_per_gpu": 235.43160351538114, "total_per_gpu": 473.369456401145, "tpot_error_pct": 3.06, "tpot_ms": 16.56051081763094, "tpot_relative": 1.400899, "ttft_error_pct": 57.85, "ttft_ms": 89.63970533749982, "ttft_relative": 0.498789, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 14916.625269687347, "interactivity_tok_s": 62.23037132756903, "output_per_gpu": 239.89529721536937, "total_per_gpu": 482.34436133639764, "tpot_ms": 16.069324007986182, "tpot_relative": 1.359349, "ttft_ms": 212.6508557872512, "ttft_relative": 1.183268, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20868.805658701687, "interactivity_tok_s": 49.689238800207605, "output_per_gpu": 393.6143801980125, "total_per_gpu": null, "tpot_error_pct": 0.33, "tpot_ms": 20.125081891892897, "tpot_relative": 1.702436, "ttft_error_pct": 1.69, "ttft_ms": 280.84688329525454, "ttft_relative": 1.562737, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18782.66312034714, "interactivity_tok_s": 49.42544887221483, "output_per_gpu": 384.49487243422, "total_per_gpu": 767.7734870381802, "tpot_error_pct": 0.2, "tpot_ms": 20.232492022185017, "tpot_relative": 1.711522, "ttft_error_pct": 62.6, "ttft_ms": 106.83408466875066, "ttft_relative": 0.594465, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 18924.24392264072, "interactivity_tok_s": 49.52380834060227, "output_per_gpu": 381.08479769701984, "total_per_gpu": 760.964124521938, "tpot_ms": 20.192308174735956, "tpot_relative": 1.708123, "ttft_ms": 285.67398732818674, "ttft_relative": 1.589596, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27732.235870160526, "interactivity_tok_s": 37.33290649065827, "output_per_gpu": 593.1266532254045, "total_per_gpu": null, "tpot_error_pct": 3.79, "tpot_ms": 26.78602053794627, "tpot_relative": 2.265904, "ttft_error_pct": 21.3, "ttft_ms": 330.13685984149345, "ttft_relative": 1.837004, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25130.321091295285, "interactivity_tok_s": 36.8729461895874, "output_per_gpu": 573.0260411975374, "total_per_gpu": 1146.3104468072286, "tpot_error_pct": 5.08, "tpot_ms": 27.120154566938055, "tpot_relative": 2.294169, "ttft_error_pct": 66.66, "ttft_ms": 139.87345671406248, "ttft_relative": 0.778308, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 24199.827586923402, "interactivity_tok_s": 38.74675115505143, "output_per_gpu": 592.8582605613998, "total_per_gpu": 1185.983827431707, "tpot_ms": 25.80861543715852, "tpot_relative": 2.183222, "ttft_ms": 419.483285698459, "ttft_relative": 2.334161, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34656.66198194031, "interactivity_tok_s": 29.80626532658575, "output_per_gpu": 948.7098720673068, "total_per_gpu": null, "tpot_error_pct": 0.93, "tpot_ms": 33.54999323273313, "tpot_relative": 2.838087, "ttft_error_pct": 40.64, "ttft_ms": 335.01890485432494, "ttft_relative": 1.86417, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31658.20298974144, "interactivity_tok_s": 29.242974399109475, "output_per_gpu": 907.5661529176459, "total_per_gpu": 1817.126082706743, "tpot_error_pct": 2.87, "tpot_ms": 34.196247835529775, "tpot_relative": 2.892755, "ttft_error_pct": 63.76, "ttft_ms": 204.5124922320312, "ttft_relative": 1.137984, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 31137.13472859441, "interactivity_tok_s": 30.08235366966699, "output_per_gpu": 920.6816698828623, "total_per_gpu": 1843.3859293185417, "tpot_ms": 33.24207975815178, "tpot_relative": 2.812039, "ttft_ms": 564.3606416554121, "ttft_relative": 3.140313, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 49984.427853663954, "interactivity_tok_s": 20.613664412152545, "output_per_gpu": 1314.748515128512, "total_per_gpu": null, "tpot_error_pct": 9.23, "tpot_ms": 48.51151061770762, "tpot_relative": 4.103723, "ttft_error_pct": 55.09, "ttft_ms": 357.1524917490575, "ttft_relative": 1.987329, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44756.2879533927, "interactivity_tok_s": 20.76455984312985, "output_per_gpu": 1291.4462456417382, "total_per_gpu": 2581.6443126759314, "tpot_error_pct": 8.44, "tpot_ms": 48.15897893115512, "tpot_relative": 4.073901, "ttft_error_pct": 57.97, "ttft_ms": 334.21946835546953, "ttft_relative": 1.859721, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 41742.63041486563, "interactivity_tok_s": 22.517159363232803, "output_per_gpu": 1380.346713309817, "total_per_gpu": 2759.359325998441, "tpot_ms": 44.41057523591774, "tpot_relative": 3.756813, "ttft_ms": 795.2856257191456, "ttft_relative": 4.425265, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.35, "tpot_shape_error_pct": 14.25, "ttft_mape_pct": 15.25, "ttft_shape_error_pct": 18.89}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.81, "tpot_shape_error_pct": 15.73, "ttft_mape_pct": 57.83, "ttft_shape_error_pct": 9.14}, "gpu": "b300", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.92, "tpot_shape_error_pct": 14.25, "ttft_mape_pct": 16.43, "ttft_shape_error_pct": 18.89}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.46, "tpot_shape_error_pct": 15.73, "ttft_mape_pct": 58.08, "ttft_shape_error_pct": 9.14}, "framework": "sglang", "id": "443d13a1cd3d8610", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11162.552834122691, "interactivity_tok_s": 92.95774985695593, "output_per_gpu": 91.22484611836127, "total_per_gpu": null, "tpot_error_pct": 9.32, "tpot_ms": 10.757575366645678, "tpot_relative": 0.906798, "ttft_error_pct": 5.99, "ttft_ms": 157.55323404416237, "ttft_relative": 0.940101, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9875.332019875112, "interactivity_tok_s": 93.53796462511896, "output_per_gpu": 90.36035073388281, "total_per_gpu": 181.60439118727484, "tpot_error_pct": 9.88, "tpot_ms": 10.690846267692434, "tpot_relative": 0.901173, "ttft_error_pct": 55.9, "ttft_ms": 73.91040770000001, "ttft_relative": 0.441014, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 11044.057942945801, "interactivity_tok_s": 84.2938680554953, "output_per_gpu": 80.54161837384495, "total_per_gpu": 161.87090301471463, "tpot_ms": 11.863259132225904, "tpot_relative": 1.0, "ttft_ms": 167.59181887391605, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13940.231141045657, "interactivity_tok_s": 74.31352580951183, "output_per_gpu": 146.44206788598567, "total_per_gpu": null, "tpot_error_pct": 0.23, "tpot_ms": 13.456500537510548, "tpot_relative": 1.1343, "ttft_error_pct": 6.43, "ttft_ms": 174.23109117236552, "ttft_relative": 1.039616, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12547.796387762564, "interactivity_tok_s": 74.4602181696707, "output_per_gpu": 145.41692476211335, "total_per_gpu": 289.7597450157855, "tpot_error_pct": 0.43, "tpot_ms": 13.42999019585632, "tpot_relative": 1.132066, "ttft_error_pct": 52.06, "ttft_ms": 78.47605901250026, "ttft_relative": 0.468257, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 12684.188980900217, "interactivity_tok_s": 74.14133146542429, "output_per_gpu": 143.32216144946662, "total_per_gpu": 285.58569110607806, "tpot_ms": 13.487753459975947, "tpot_relative": 1.136935, "ttft_ms": 163.70218281226698, "ttft_relative": 0.976791, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17306.924195555843, "interactivity_tok_s": 59.82433273122517, "output_per_gpu": 236.5206636702564, "total_per_gpu": null, "tpot_error_pct": 2.54, "tpot_ms": 16.715606415415184, "tpot_relative": 1.409023, "ttft_error_pct": 0.31, "ttft_ms": 206.85883258610914, "ttft_relative": 1.234301, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15429.533349093701, "interactivity_tok_s": 59.652701015253825, "output_per_gpu": 232.73449628872385, "total_per_gpu": 467.9465303254835, "tpot_error_pct": 2.84, "tpot_ms": 16.76370026806145, "tpot_relative": 1.413077, "ttft_error_pct": 57.81, "ttft_ms": 87.00550593124998, "ttft_relative": 0.519151, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 15126.968648540787, "interactivity_tok_s": 61.345742817539104, "output_per_gpu": 236.66266658943366, "total_per_gpu": 475.8446876337339, "tpot_ms": 16.301049658397716, "tpot_relative": 1.374079, "ttft_ms": 206.22259184601717, "ttft_relative": 1.230505, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20609.50012841214, "interactivity_tok_s": 50.29930085163127, "output_per_gpu": 398.5515334733457, "total_per_gpu": null, "tpot_error_pct": 1.53, "tpot_ms": 19.88099204300508, "tpot_relative": 1.675846, "ttft_error_pct": 1.24, "ttft_ms": 271.2452684179456, "ttft_relative": 1.618488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18580.911475140616, "interactivity_tok_s": 49.95577582289597, "output_per_gpu": 388.62417585346, "total_per_gpu": 776.0190318100965, "tpot_error_pct": 0.85, "tpot_ms": 20.017705330915412, "tpot_relative": 1.68737, "ttft_error_pct": 62.32, "ttft_ms": 103.47667287187514, "ttft_relative": 0.617433, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 18908.871990573243, "interactivity_tok_s": 49.52963293986999, "output_per_gpu": 381.364166940557, "total_per_gpu": 761.5219792910375, "tpot_ms": 20.189933594178274, "tpot_relative": 1.701888, "ttft_ms": 274.64705735765165, "ttft_relative": 1.638786, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27745.264678248113, "interactivity_tok_s": 37.30067808095504, "output_per_gpu": 592.7905993537639, "total_per_gpu": null, "tpot_error_pct": 5.19, "tpot_ms": 26.809164107678235, "tpot_relative": 2.259848, "ttft_error_pct": 13.06, "ttft_ms": 319.48979609327944, "ttft_relative": 1.906357, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25202.548901363996, "interactivity_tok_s": 36.76043234116158, "output_per_gpu": 571.5426106729569, "total_per_gpu": 1143.3429169129752, "tpot_error_pct": 6.73, "tpot_ms": 27.203162104279027, "tpot_relative": 2.29306, "ttft_error_pct": 62.94, "ttft_ms": 136.1831430937497, "ttft_relative": 0.812588, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 23844.365386327263, "interactivity_tok_s": 39.235918061415305, "output_per_gpu": 601.4932769633318, "total_per_gpu": 1203.2577535681182, "tpot_ms": 25.48685106423959, "tpot_relative": 2.148385, "ttft_ms": 367.50410350359743, "ttft_relative": 2.192852, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34155.569261680495, "interactivity_tok_s": 30.238871424899685, "output_per_gpu": 962.5915497616699, "total_per_gpu": null, "tpot_error_pct": 3.02, "tpot_ms": 33.070017261840235, "tpot_relative": 2.7876, "ttft_error_pct": 35.49, "ttft_ms": 324.9416028179356, "ttft_relative": 1.938887, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31286.6929931937, "interactivity_tok_s": 29.58803881443854, "output_per_gpu": 918.1712701936628, "total_per_gpu": 1838.3596149957805, "tpot_error_pct": 5.29, "tpot_ms": 33.79744113056977, "tpot_relative": 2.848917, "ttft_error_pct": 60.26, "ttft_ms": 200.15414613203112, "ttft_relative": 1.194295, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 30019.3256229094, "interactivity_tok_s": 31.15223663989363, "output_per_gpu": 954.084392190033, "total_per_gpu": 1910.2647543415346, "tpot_ms": 32.100423849483654, "tpot_relative": 2.705869, "ttft_ms": 503.699884503203, "ttft_relative": 3.005516, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 49249.994882443105, "interactivity_tok_s": 20.91969829991331, "output_per_gpu": 1334.332990673912, "total_per_gpu": null, "tpot_error_pct": 12.63, "tpot_ms": 47.80183660699083, "tpot_relative": 4.029402, "ttft_error_pct": 52.49, "ttft_ms": 348.7160334914834, "ttft_relative": 2.080746, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44254.50081836311, "interactivity_tok_s": 20.998629392282748, "output_per_gpu": 1305.8896288708695, "total_per_gpu": 2610.517119651151, "tpot_error_pct": 12.2, "tpot_ms": 47.622155775915175, "tpot_relative": 4.014256, "ttft_error_pct": 55.29, "ttft_ms": 328.16960517304733, "ttft_relative": 1.958148, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 39859.87687791894, "interactivity_tok_s": 23.561042496652828, "output_per_gpu": 1444.30165723849, "total_per_gpu": 2887.2074016099205, "tpot_ms": 42.44294369156474, "tpot_relative": 3.57768, "ttft_ms": 734.0102099723026, "ttft_relative": 4.37975, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.31, "tpot_shape_error_pct": null, "ttft_mape_pct": 6.97, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.29, "tpot_shape_error_pct": null, "ttft_mape_pct": 56.06, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "672e49ec52facbb0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9413.315481099833, "interactivity_tok_s": 110.34877403660076, "output_per_gpu": 54.06219633801939, "total_per_gpu": null, "tpot_error_pct": 16.31, "tpot_ms": 9.062175893937141, "tpot_relative": 0.836932, "ttft_error_pct": 6.97, "ttft_ms": 142.70954160213813, "ttft_relative": 0.930307, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8377.510382524837, "interactivity_tok_s": 110.32409673014413, "output_per_gpu": 53.11608061423418, "total_per_gpu": 106.75161621063921, "tpot_error_pct": 16.29, "tpot_ms": 9.064202922467866, "tpot_relative": 0.837119, "ttft_error_pct": 56.06, "ttft_ms": 67.39939354999981, "ttft_relative": 0.439369, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 10080.613710117177, "interactivity_tok_s": 92.35444099625013, "output_per_gpu": 44.09043793832725, "total_per_gpu": 88.61206352055271, "tpot_ms": 10.827849632489281, "tpot_relative": 1.0, "ttft_ms": 153.40044663462322, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 15.38, "points": 2, "status_counts": {"failed": 11, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 53.84, "tpot_shape_error_pct": 1.65, "ttft_mape_pct": 78.23, "ttft_shape_error_pct": 83.83}, "aisimulate": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.16, "tpot_shape_error_pct": 7.28, "ttft_mape_pct": 32.64, "ttft_shape_error_pct": 73.58}, "gpu": "gb200", "precisions": ["fp4"], "rows": 13, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 53.84, "tpot_shape_error_pct": 1.65, "ttft_mape_pct": 78.23, "ttft_shape_error_pct": 83.83}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.18, "tpot_shape_error_pct": 0.76, "ttft_mape_pct": 15.44, "ttft_shape_error_pct": 32.64}, "framework": "dynamo-trt", "id": "ce295e2f7834383d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 16348.7, "interactivity_tok_s": 63.25110689437065, "output_per_gpu": 12.168541666666668, "total_per_gpu": null, "tpot_error_pct": 52.58, "tpot_ms": 15.81, "tpot_relative": 1.525782, "ttft_error_pct": 62.52, "ttft_ms": 175.07, "ttft_relative": 0.37477, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12833.728019900032, "interactivity_tok_s": 75.0074129102684, "output_per_gpu": 14.756757239988476, "total_per_gpu": 29.4045154792941, "tpot_error_pct": 28.66, "tpot_ms": 13.332015612860866, "tpot_relative": 1.286638, "ttft_error_pct": 5.38, "ttft_ms": 441.9917148249998, "ttft_relative": 0.946167, "unavailable_metrics": {}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 10083.612586720847, "interactivity_tok_s": 96.50739929959776, "output_per_gpu": 22.545019554764004, "total_per_gpu": 37.4564826202988, "tpot_ms": 10.361899784446559, "tpot_relative": 1.0, "ttft_ms": 467.13929549441673, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19890.326, "interactivity_tok_s": 51.88875051888751, "output_per_gpu": 49.85175, "total_per_gpu": null, "tpot_error_pct": 55.1, "tpot_ms": 19.272, "tpot_relative": 1.859891, "ttft_error_pct": 93.94, "ttft_ms": 175.07, "ttft_relative": 0.37477, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18311.619540870208, "interactivity_tok_s": 63.02824306713745, "output_per_gpu": 51.27298613949957, "total_per_gpu": 102.27210857877493, "tpot_error_pct": 27.69, "tpot_ms": 15.865903146543426, "tpot_relative": 1.531177, "ttft_error_pct": 25.5, "ttft_ms": 3626.55146248501, "ttft_relative": 7.763319, "unavailable_metrics": {}}, "concurrency": 25, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 14388.262027036399, "interactivity_tok_s": 80.4805760800732, "output_per_gpu": 78.49737717210311, "total_per_gpu": 130.50643528572883, "tpot_ms": 12.425358374734566, "tpot_relative": 1.199139, "ttft_ms": 2889.6619507099967, "ttft_relative": 6.185868, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 65.39, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 59.92, "ttft_shape_error_pct": 2.17}, "framework": "dynamo-trt", "id": "0b75b5fe8f5bf430", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 36874.43092878563, "interactivity_tok_s": 32.533169400421926, "output_per_gpu": 445.95435403102823, "total_per_gpu": 891.9024885336067, "tpot_error_pct": 61.54, "tpot_ms": 30.7378598036941, "tpot_relative": 1.615428, "ttft_error_pct": 58.2, "ttft_ms": 8568.258631811012, "ttft_relative": 1.582041, "unavailable_metrics": {}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 22937.16826788778, "interactivity_tok_s": 52.55497793983208, "output_per_gpu": 809.6696043915082, "total_per_gpu": 1439.907844123839, "tpot_ms": 19.02769326903451, "tpot_relative": 1.0, "ttft_ms": 5415.952087367056, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 39338.41212393753, "interactivity_tok_s": 28.400624629388354, "output_per_gpu": 772.1121196083245, "total_per_gpu": 1543.6555731969636, "tpot_error_pct": 69.23, "tpot_ms": 35.210493186309066, "tpot_relative": 1.850487, "ttft_error_pct": 61.63, "ttft_ms": 6892.151705876296, "ttft_relative": 1.272565, "unavailable_metrics": {}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 23435.749780816233, "interactivity_tok_s": 48.06355010314757, "output_per_gpu": 1460.699642473873, "total_per_gpu": 2596.6975787402243, "tpot_ms": 20.805787293155284, "tpot_relative": 1.093448, "ttft_ms": 4264.171131521264, "ttft_relative": 0.787335, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 65.02, "tpot_shape_error_pct": 7.13, "ttft_mape_pct": 40.41, "ttft_shape_error_pct": 3.24}, "framework": "dynamo-trt", "id": "254e825153a89016", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41293.39338094265, "interactivity_tok_s": 23.720042131604888, "output_per_gpu": 1938.661684878406, "total_per_gpu": 3878.70612274322, "tpot_error_pct": 59.34, "tpot_ms": 42.15844113816253, "tpot_relative": 1.593391, "ttft_error_pct": 42.72, "ttft_ms": 2488.9320685742637, "ttft_relative": 1.427178, "unavailable_metrics": {}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 26095.68564081236, "interactivity_tok_s": 37.7952996215858, "output_per_gpu": 4606.40826870825, "total_per_gpu": 6146.01714665989, "tpot_ms": 26.458316510576783, "tpot_relative": 1.0, "ttft_ms": 1743.9534356304803, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 60863.846426385055, "interactivity_tok_s": 16.156930895050426, "output_per_gpu": 2647.3172903123464, "total_per_gpu": 5295.248132546664, "tpot_error_pct": 70.7, "tpot_ms": 61.8929428178927, "tpot_relative": 2.339262, "ttft_error_pct": 38.09, "ttft_ms": 3888.269854071795, "ttft_relative": 2.229572, "unavailable_metrics": {}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 36194.58487603828, "interactivity_tok_s": 27.579181565236723, "output_per_gpu": 6668.942011929536, "total_per_gpu": 8895.01017062333, "tpot_ms": 36.259234076057204, "tpot_relative": 1.370429, "ttft_ms": 2815.6882903990027, "ttft_relative": 1.614543, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 84.15, "tpot_shape_error_pct": null, "ttft_mape_pct": 54.06, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "0b1abb19f9b10dfb", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41573.422537053986, "interactivity_tok_s": 23.579320585672246, "output_per_gpu": 1157.8743883092907, "total_per_gpu": 2316.574632042871, "tpot_error_pct": 84.15, "tpot_ms": 42.410043002156755, "tpot_relative": 1.841544, "ttft_error_pct": 54.06, "ttft_ms": 2537.4006376456314, "ttft_relative": 1.540641, "unavailable_metrics": {}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 22843.03993517615, "interactivity_tok_s": 43.422366184551855, "output_per_gpu": 2629.4644778630377, "total_per_gpu": 4209.978662272033, "tpot_ms": 23.029606349636577, "tpot_relative": 1.0, "ttft_ms": 1646.9768309983292, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.5, "tpot_shape_error_pct": 9.59, "ttft_mape_pct": 23.1, "ttft_shape_error_pct": 119.25}, "framework": "dynamo-trt", "id": "9adcde7941fab50c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12626.92213448438, "interactivity_tok_s": 75.4657835003552, "output_per_gpu": 7.980753858407842, "total_per_gpu": 15.949947056676166, "tpot_error_pct": 32.67, "tpot_ms": 13.251038465602006, "tpot_relative": 1.326738, "ttft_error_pct": 46.53, "ttft_ms": 334.01823389062076, "ttft_relative": 0.534735, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 9885.688545415178, "interactivity_tok_s": 100.12335543864013, "output_per_gpu": 11.163480767767702, "total_per_gpu": 19.842510892535543, "tpot_ms": 9.987679653952895, "tpot_relative": 1.0, "ttft_ms": 624.6421522082528, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16156.52998307501, "interactivity_tok_s": 71.45740540071621, "output_per_gpu": 30.833241145979198, "total_per_gpu": 61.568948687053165, "tpot_error_pct": 20.09, "tpot_ms": 13.99435082189504, "tpot_relative": 1.401161, "ttft_error_pct": 16.07, "ttft_ms": 3241.556784915626, "ttft_relative": 5.189462, "unavailable_metrics": {}}, "concurrency": 20, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 13546.13629704836, "interactivity_tok_s": 85.8130985016784, "output_per_gpu": 40.92323652479457, "total_per_gpu": 72.63922736334493, "tpot_ms": 11.653232635346937, "tpot_relative": 1.166761, "ttft_ms": 2792.6920414101915, "ttft_relative": 4.470867, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19804.224817833983, "interactivity_tok_s": 60.66942433096826, "output_per_gpu": 104.59979335774268, "total_per_gpu": 209.49719668844818, "tpot_error_pct": 21.16, "tpot_ms": 16.482767242766755, "tpot_relative": 1.65031, "ttft_error_pct": 20.03, "ttft_ms": 4659.819384726906, "ttft_relative": 7.459982, "unavailable_metrics": {}}, "concurrency": 84, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 16382.954979736365, "interactivity_tok_s": 73.50651819936546, "output_per_gpu": 142.6373179000281, "total_per_gpu": 253.9840750188816, "tpot_ms": 13.604235712645039, "tpot_relative": 1.362102, "ttft_ms": 3882.293410170643, "ttft_relative": 6.215228, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22963.385812732886, "interactivity_tok_s": 52.522392021221464, "output_per_gpu": 181.2981786326362, "total_per_gpu": 362.33712732810017, "tpot_error_pct": 19.94, "tpot_ms": 19.039498421853178, "tpot_relative": 1.906298, "ttft_error_pct": 18.93, "ttft_ms": 5420.496584136158, "ttft_relative": 8.677763, "unavailable_metrics": {}}, "concurrency": 168, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 19182.42934312643, "interactivity_tok_s": 62.9937769003753, "output_per_gpu": 244.26073186433587, "total_per_gpu": 434.0829569473806, "tpot_ms": 15.874583954245205, "tpot_relative": 1.589417, "ttft_ms": 4557.592010877886, "ttft_relative": 7.296325, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22220.84651085376, "interactivity_tok_s": 46.202024571298196, "output_per_gpu": 317.3451677741748, "total_per_gpu": 634.6523792521698, "tpot_error_pct": 18.64, "tpot_ms": 21.644073160837717, "tpot_relative": 2.167077, "ttft_error_pct": 13.92, "ttft_ms": 2289.7745449768904, "ttft_relative": 3.665738, "unavailable_metrics": {}}, "concurrency": 284, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 18807.996947585172, "interactivity_tok_s": 54.81470171464606, "output_per_gpu": 422.4047061131857, "total_per_gpu": 751.1717184099332, "tpot_ms": 18.24328088485808, "tpot_relative": 1.826578, "ttft_ms": 2009.9161480833009, "ttft_relative": 3.217708, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 23.23, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "4be0819a29096fb3", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 78817.88569481079, "interactivity_tok_s": 12.695778855323523, "output_per_gpu": 4100.770914903848, "total_per_gpu": 8202.492239449039, "tpot_error_pct": 34.28, "tpot_ms": 78.7663373311426, "tpot_relative": 1.342818, "ttft_error_pct": 23.23, "ttft_ms": 6309.767511430721, "ttft_relative": 1.232257, "unavailable_metrics": {}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 59117.56871578042, "interactivity_tok_s": 17.04811524921923, "output_per_gpu": 8208.943524135038, "total_per_gpu": 10949.058487333748, "tpot_ms": 58.657510544797496, "tpot_relative": 1.0, "ttft_ms": 5120.494661527859, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}]}, {"aic": {"coverage_pct": 13.64, "points": 3, "status_counts": {"failed": 19, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 69.88, "tpot_shape_error_pct": 45.66, "ttft_mape_pct": 85.94, "ttft_shape_error_pct": 86.75}, "aisimulate": {"coverage_pct": 100.0, "points": 22, "status_counts": {"failed": 0, "success": 22, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.25, "tpot_shape_error_pct": 27.82, "ttft_mape_pct": 59.57, "ttft_shape_error_pct": 52.06}, "gpu": "gb300", "precisions": ["fp4"], "rows": 22, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.3, "tpot_shape_error_pct": null, "ttft_mape_pct": 2.56, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "2c8d7f259b8dfea4", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 79393.73699931288, "interactivity_tok_s": 24.212758746723747, "output_per_gpu": 2310.212024809163, "total_per_gpu": 4620.927895442637, "tpot_error_pct": 40.3, "tpot_ms": 41.3005395403492, "tpot_relative": 1.403, "ttft_error_pct": 2.56, "ttft_ms": 41374.44969453783, "ttft_relative": 0.974432, "unavailable_metrics": {}}, "concurrency": 8300, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 69557.99646867582, "interactivity_tok_s": 33.97049331509609, "output_per_gpu": 3267.946301268811, "total_per_gpu": 5230.877470318107, "tpot_ms": 29.43731169060214, "tpot_relative": 1.0, "ttft_ms": 42460.065067181815, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.18, "tpot_shape_error_pct": 19.97, "ttft_mape_pct": 62.42, "ttft_shape_error_pct": 36.44}, "framework": "dynamo-sglang", "id": "da97148065832fd8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13414.446693668771, "interactivity_tok_s": 127.05321735107307, "output_per_gpu": 28.718623872577798, "total_per_gpu": 57.34640322767694, "tpot_error_pct": 30.32, "tpot_ms": 7.870717647683041, "tpot_relative": 0.696848, "ttft_error_pct": 46.96, "ttft_ms": 6151.414143984379, "ttft_relative": 0.530353, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 22022.68146679371, "interactivity_tok_s": 88.53677924992527, "output_per_gpu": 17.85094517141223, "total_per_gpu": 33.66599054749457, "tpot_ms": 11.29474110614707, "tpot_relative": 1.0, "ttft_ms": 11598.716703990567, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9314.203083395298, "interactivity_tok_s": 101.76821067161781, "output_per_gpu": 83.89815422698469, "total_per_gpu": 167.83413622398194, "tpot_error_pct": 25.32, "tpot_ms": 9.826251178049754, "tpot_relative": 0.869985, "ttft_error_pct": 69.21, "ttft_ms": 265.7170954281231, "ttft_relative": 0.022909, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 12974.349785319078, "interactivity_tok_s": 76.00516928854834, "output_per_gpu": 64.32216575884156, "total_per_gpu": 121.5854904536747, "tpot_ms": 13.156999837781685, "tpot_relative": 1.164878, "ttft_ms": 862.8873249907883, "ttft_relative": 0.074395, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11383.244346282812, "interactivity_tok_s": 83.22227923754329, "output_per_gpu": 138.30901036638764, "total_per_gpu": 276.9218633839091, "tpot_error_pct": 16.1, "tpot_ms": 12.016013129677411, "tpot_relative": 1.063859, "ttft_error_pct": 70.87, "ttft_ms": 341.24861988984395, "ttft_relative": 0.029421, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 14331.928359242273, "interactivity_tok_s": 69.82630766475143, "output_per_gpu": 115.5871019476992, "total_per_gpu": 218.62688733786828, "tpot_ms": 14.321249876209674, "tpot_relative": 1.267957, "ttft_ms": 1171.305654025923, "ttft_relative": 0.100986, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13872.745326919923, "interactivity_tok_s": 68.78608487551881, "output_per_gpu": 227.30010089875324, "total_per_gpu": 454.3805169872466, "tpot_error_pct": 11.6, "tpot_ms": 14.537824064411947, "tpot_relative": 1.287132, "ttft_error_pct": 66.19, "ttft_ms": 484.66372409101507, "ttft_relative": 0.041786, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 16578.7138300915, "interactivity_tok_s": 60.80447807704637, "output_per_gpu": 200.7557636862825, "total_per_gpu": 379.1466842932067, "tpot_ms": 16.446157119100395, "tpot_relative": 1.45609, "ttft_ms": 1433.303850670353, "ttft_relative": 0.123574, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16461.7954895683, "interactivity_tok_s": 58.63874342414763, "output_per_gpu": 383.9858397378193, "total_per_gpu": 767.6179597646178, "tpot_error_pct": 12.59, "tpot_ms": 17.0535714376887, "tpot_relative": 1.509868, "ttft_error_pct": 58.9, "ttft_ms": 751.5065347490241, "ttft_relative": 0.064792, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 19801.570602714415, "interactivity_tok_s": 51.25717926539979, "output_per_gpu": 338.2902646947306, "total_per_gpu": 638.912142974997, "tpot_ms": 19.50946217352681, "tpot_relative": 1.727305, "ttft_ms": 1828.4536256739982, "ttft_relative": 0.157643, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 206.21, "tpot_shape_error_pct": null, "ttft_mape_pct": 87.9, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "de1eba16475d967f", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 112368.23543690638, "interactivity_tok_s": 9.353294917590269, "output_per_gpu": 3004.111584274724, "total_per_gpu": 6009.218459905334, "tpot_error_pct": 206.21, "tpot_ms": 106.91419535156008, "tpot_relative": 3.062104, "ttft_error_pct": 87.9, "ttft_ms": 13965.190463994722, "ttft_relative": 0.120955, "unavailable_metrics": {}}, "concurrency": 16500, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 147597.13930273923, "interactivity_tok_s": 28.64076383090822, "output_per_gpu": 3117.2399643899907, "total_per_gpu": 4536.074697534437, "tpot_ms": 34.9152699244994, "tpot_relative": 1.0, "ttft_ms": 115458.06958309561, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.83, "tpot_shape_error_pct": 26.32, "ttft_mape_pct": 41.15, "ttft_shape_error_pct": 63.68}, "framework": "dynamo-sglang", "id": "263089c69d62131e", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17943.57250250233, "interactivity_tok_s": 52.79238831435117, "output_per_gpu": 349.4851383039339, "total_per_gpu": 698.6325003552689, "tpot_error_pct": 7.03, "tpot_ms": 18.94212464959003, "tpot_relative": 0.929677, "ttft_error_pct": 58.38, "ttft_ms": 499.1332144750004, "ttft_relative": 0.416173, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 19962.333621277, "interactivity_tok_s": 49.079857913793305, "output_per_gpu": 353.78675299195845, "total_per_gpu": 628.8569547780253, "tpot_ms": 20.374957110846932, "tpot_relative": 1.0, "ttft_ms": 1199.3398593110242, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 21920.48673410451, "interactivity_tok_s": 43.54057822401511, "output_per_gpu": 575.066885091122, "total_per_gpu": 1149.604030615413, "tpot_error_pct": 8.4, "tpot_ms": 22.96708130183818, "tpot_relative": 1.127221, "ttft_error_pct": 52.44, "ttft_ms": 761.9675178835938, "ttft_relative": 0.635322, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 21122.183984318715, "interactivity_tok_s": 47.1979797584854, "output_per_gpu": 667.1428106594658, "total_per_gpu": 1185.8821006369749, "tpot_ms": 21.187347533031154, "tpot_relative": 1.039872, "ttft_ms": 1602.1498766825957, "ttft_relative": 1.33586, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26258.472670580773, "interactivity_tok_s": 36.89529400674862, "output_per_gpu": 958.2495485817458, "total_per_gpu": 1916.442661619717, "tpot_error_pct": 21.8, "tpot_ms": 27.103727641175244, "tpot_relative": 1.330247, "ttft_error_pct": 48.48, "ttft_ms": 1298.4245902927746, "ttft_relative": 1.082616, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 23010.35411962774, "interactivity_tok_s": 44.93986402765204, "output_per_gpu": 1222.3181570956156, "total_per_gpu": 2173.837877254344, "tpot_ms": 22.25195873722911, "tpot_relative": 1.092123, "ttft_ms": 2520.3558213733595, "ttft_relative": 2.101453, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 48206.27150440504, "interactivity_tok_s": 34.72991811154252, "output_per_gpu": 1270.8823799653444, "total_per_gpu": 2541.0382821210787, "tpot_error_pct": 22.1, "tpot_ms": 28.793618136048792, "tpot_relative": 1.413187, "ttft_error_pct": 5.29, "ttft_ms": 21675.728771958366, "ttft_relative": 18.07305, "unavailable_metrics": {}}, "concurrency": 2500, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 42315.08676495206, "interactivity_tok_s": 42.406282726277674, "output_per_gpu": 1622.0140468395123, "total_per_gpu": 2883.6232316238415, "tpot_ms": 23.581411425631405, "tpot_relative": 1.157372, "ttft_ms": 20587.471340868025, "ttft_relative": 17.165669, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 78.59, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "28a7e8b983df2a30", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 30744.212472410745, "interactivity_tok_s": 31.1195693671036, "output_per_gpu": 560.968960592437, "total_per_gpu": 1122.013867865454, "tpot_error_pct": 72.56, "tpot_ms": 32.13412075865988, "tpot_relative": 1.725633, "ttft_error_pct": 78.59, "ttft_ms": 1160.1853696861842, "ttft_relative": 0.214095, "unavailable_metrics": {}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 22566.29105280967, "interactivity_tok_s": 53.70096474899331, "output_per_gpu": 822.2626327808314, "total_per_gpu": 1548.3209848508002, "tpot_ms": 18.621639381604336, "tpot_relative": 1.0, "ttft_ms": 5419.021865952021, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 34}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 120.9, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.01, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 78.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 12.92, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "3e608c37cc60785b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 124421.261, "interactivity_tok_s": 8.234519104084322, "output_per_gpu": 2588.4040416666667, "total_per_gpu": null, "tpot_error_pct": 120.9, "tpot_ms": 121.44, "tpot_relative": 2.208987, "ttft_error_pct": 93.01, "ttft_ms": 188.141, "ttft_relative": 0.069877, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 93376.22779572208, "interactivity_tok_s": 10.190349231592624, "output_per_gpu": 3268.965320352305, "total_per_gpu": 6538.615773454465, "tpot_error_pct": 78.5, "tpot_ms": 98.13206370786102, "tpot_relative": 1.785017, "ttft_error_pct": 12.92, "ttft_ms": 3040.449339122554, "ttft_relative": 1.129243, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 53294.96316289202, "interactivity_tok_s": 18.18994475487472, "output_per_gpu": 8661.523437419288, "total_per_gpu": 11552.96351713057, "tpot_ms": 54.975428099198055, "tpot_relative": 1.0, "ttft_ms": 2692.4662741751354, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 77.07, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.13, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "f91ee9b7c0c6f86b", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 38780.88264595307, "interactivity_tok_s": 24.547200474677595, "output_per_gpu": 1349.6749640826536, "total_per_gpu": 2698.313661571232, "tpot_error_pct": 77.07, "tpot_ms": 40.73784303964031, "tpot_relative": 1.770653, "ttft_error_pct": 53.13, "ttft_ms": 1240.472748680857, "ttft_relative": 0.468689, "unavailable_metrics": {}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 23823.240988636895, "interactivity_tok_s": 43.464570307616285, "output_per_gpu": 2636.7243488496947, "total_per_gpu": 4443.804019629844, "tpot_ms": 23.007244588468193, "tpot_relative": 1.0, "ttft_ms": 2646.685161954178, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 38}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 84.27, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.27, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "e9fe1aff5b4070b9", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 60068.95530608723, "interactivity_tok_s": 15.899307394761495, "output_per_gpu": 2892.1691247290514, "total_per_gpu": 5786.342275348895, "tpot_error_pct": 84.27, "tpot_ms": 62.89582150789034, "tpot_relative": 1.842746, "ttft_error_pct": 20.27, "ttft_ms": 2177.9839559762586, "ttft_relative": 0.797338, "unavailable_metrics": {}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 34151.89091640228, "interactivity_tok_s": 29.298384568143465, "output_per_gpu": 7060.59274348473, "total_per_gpu": 10273.519054568993, "tpot_ms": 34.13157464959053, "tpot_relative": 1.0, "ttft_ms": 2731.5703451068116, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.37, "tpot_shape_error_pct": 45.66, "ttft_mape_pct": 82.41, "ttft_shape_error_pct": 86.75}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.54, "tpot_shape_error_pct": 38.06, "ttft_mape_pct": 86.76, "ttft_shape_error_pct": 64.71}, "framework": "dynamo-trt", "id": "57b8ba004f80f8b7", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12353.657, "interactivity_tok_s": 84.09014463504877, "output_per_gpu": 17.556954545454545, "total_per_gpu": null, "tpot_error_pct": 17.53, "tpot_ms": 11.892, "tpot_relative": 1.175316, "ttft_error_pct": 68.93, "ttft_ms": 188.141, "ttft_relative": 0.310708, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10205.345997639995, "interactivity_tok_s": 91.51874428677817, "output_per_gpu": 20.26339352553866, "total_per_gpu": 40.60969351893736, "tpot_error_pct": 7.99, "tpot_ms": 10.9267233482406, "tpot_relative": 1.079916, "ttft_error_pct": 80.43, "ttft_ms": 118.52288781999819, "ttft_relative": 0.195736, "unavailable_metrics": {}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 9995.57343252527, "interactivity_tok_s": 98.8325184794148, "output_per_gpu": 22.744977405235424, "total_per_gpu": 41.22403009395954, "tpot_ms": 10.118127266060549, "tpot_relative": 1.0, "ttft_ms": 605.5229884805158, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28245.962, "interactivity_tok_s": 36.46042221168921, "output_per_gpu": 160.1535909090909, "total_per_gpu": null, "tpot_error_pct": 71.2, "tpot_ms": 27.427, "tpot_relative": 2.710679, "ttft_error_pct": 95.88, "ttft_ms": 188.141, "ttft_relative": 0.310708, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22298.04361565228, "interactivity_tok_s": 41.866210799682705, "output_per_gpu": 190.39492377157646, "total_per_gpu": 381.1047600550508, "tpot_error_pct": 49.1, "tpot_ms": 23.885610397957937, "tpot_relative": 2.360675, "ttft_error_pct": 93.09, "ttft_ms": 315.60046534857076, "ttft_relative": 0.521203, "unavailable_metrics": {}}, "concurrency": 105, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 19281.60580634763, "interactivity_tok_s": 62.42058328522514, "output_per_gpu": 243.41081027948525, "total_per_gpu": 443.4285633609876, "tpot_ms": 16.0203565453817, "tpot_relative": 1.583332, "ttft_ms": 4568.705234653178, "ttft_relative": 7.545057, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 73.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.17, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "f06a18f396469c48", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 43609.365320511846, "interactivity_tok_s": 22.037005422309377, "output_per_gpu": 2281.8883120933965, "total_per_gpu": 4562.024613745643, "tpot_error_pct": 73.42, "tpot_ms": 45.378216360905384, "tpot_relative": 1.734237, "ttft_error_pct": 43.17, "ttft_ms": 1793.186625359325, "ttft_relative": 0.568319, "unavailable_metrics": {}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 27239.20919583982, "interactivity_tok_s": 38.217379660805754, "output_per_gpu": 4616.849442088915, "total_per_gpu": 7391.957908624201, "tpot_ms": 26.16610581037718, "tpot_relative": 1.0, "ttft_ms": 3155.2440073868615, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.7, "tpot_shape_error_pct": 36.39, "ttft_mape_pct": 90.44, "ttft_shape_error_pct": 57.05}, "framework": "dynamo-trt", "id": "43d5f6f1c55e1f89", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9088.719969725009, "interactivity_tok_s": 102.09196845890949, "output_per_gpu": 11.47640385442687, "total_per_gpu": 23.065042555451015, "tpot_error_pct": 0.76, "tpot_ms": 9.795089810639563, "tpot_relative": 0.992366, "ttft_error_pct": 83.29, "ttft_ms": 108.12037904999897, "ttft_relative": 0.167091, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 9796.339052660187, "interactivity_tok_s": 101.31262395694898, "output_per_gpu": 11.245243217985541, "total_per_gpu": 21.16359468831718, "tpot_ms": 9.870438262707838, "tpot_relative": 1.0, "ttft_ms": 647.0759862968407, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13161.019917358337, "interactivity_tok_s": 70.59269907687343, "output_per_gpu": 48.01982978909015, "total_per_gpu": 96.25303648443763, "tpot_error_pct": 24.79, "tpot_ms": 14.165770866913993, "tpot_relative": 1.435171, "ttft_error_pct": 96.87, "ttft_ms": 165.48525878333285, "ttft_relative": 0.255743, "unavailable_metrics": {}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 15806.858449880261, "interactivity_tok_s": 88.09602662546537, "output_per_gpu": 41.61751260602608, "total_per_gpu": 78.10388701033156, "tpot_ms": 11.351249747635453, "tpot_relative": 1.150025, "ttft_ms": 5285.5712816908635, "ttft_relative": 8.168394, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17959.5362886446, "interactivity_tok_s": 52.13842944452558, "output_per_gpu": 134.26936136044515, "total_per_gpu": 268.6916548705894, "tpot_error_pct": 42.75, "tpot_ms": 19.179710832371416, "tpot_relative": 1.943147, "ttft_error_pct": 94.44, "ttft_ms": 296.88178425543236, "ttft_relative": 0.458805, "unavailable_metrics": {}}, "concurrency": 92, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 17688.756930283413, "interactivity_tok_s": 74.42526744969662, "output_per_gpu": 145.08262562071945, "total_per_gpu": 273.45978714026387, "tpot_ms": 13.436297030116704, "tpot_relative": 1.361267, "ttft_ms": 5337.275482337566, "ttft_relative": 8.248298, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 23764.47762159236, "interactivity_tok_s": 39.92341060253677, "output_per_gpu": 367.755895947361, "total_per_gpu": 734.823040909067, "tpot_error_pct": 38.51, "tpot_ms": 25.04796020449363, "tpot_relative": 2.537675, "ttft_error_pct": 87.16, "ttft_ms": 678.3090550476193, "ttft_relative": 1.048268, "unavailable_metrics": {}}, "concurrency": 336, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 21936.264843350553, "interactivity_tok_s": 55.29970504441459, "output_per_gpu": 429.30233312351334, "total_per_gpu": 808.2663676661026, "tpot_ms": 18.083279091576323, "tpot_relative": 1.832064, "ttft_ms": 5283.285907101561, "ttft_relative": 8.164862, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 34}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 51}, {"aic": {"coverage_pct": 48.94, "points": 23, "status_counts": {"failed": 24, "success": 23, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.95, "tpot_shape_error_pct": 11.68, "ttft_mape_pct": 47.26, "ttft_shape_error_pct": 54.68}, "aisimulate": {"coverage_pct": 100.0, "points": 47, "status_counts": {"failed": 0, "success": 47, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.95, "tpot_shape_error_pct": 15.2, "ttft_mape_pct": 41.7, "ttft_shape_error_pct": 41.32}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 75.0, "points": 6, "status_counts": {"failed": 2, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.07, "tpot_shape_error_pct": 11.14, "ttft_mape_pct": 20.91, "ttft_shape_error_pct": 30.13}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.62, "tpot_shape_error_pct": 6.26, "ttft_mape_pct": 46.93, "ttft_shape_error_pct": 60.23}, "gpu": "b200", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.65, "tpot_shape_error_pct": null, "ttft_mape_pct": 26.36, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.48, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.7, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "1d5c37d604fbf620", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10927.154471312446, "interactivity_tok_s": 99.31550160187503, "output_per_gpu": 45.68018875049332, "total_per_gpu": null, "tpot_error_pct": 17.65, "tpot_ms": 10.068921607109122, "tpot_relative": 0.823543, "ttft_error_pct": 26.36, "ttft_ms": 626.6476672398157, "ttft_relative": 1.26356, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10036.80009229999, "interactivity_tok_s": 94.53857231100942, "output_per_gpu": 44.76526932780666, "total_per_gpu": 402.4130738609284, "tpot_error_pct": 13.48, "tpot_ms": 10.577693057498667, "tpot_relative": 0.865156, "ttft_error_pct": 36.7, "ttft_ms": 313.9208578750005, "ttft_relative": 0.632984, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 11732.783799275603, "interactivity_tok_s": 81.79061325836727, "output_per_gpu": 38.2445144383994, "total_per_gpu": 343.79537629443377, "tpot_ms": 12.226341876678603, "tpot_relative": 1.0, "ttft_ms": 495.93816615069954, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 71.43, "points": 5, "status_counts": {"failed": 2, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.56, "tpot_shape_error_pct": 11.14, "ttft_mape_pct": 19.82, "ttft_shape_error_pct": 30.13}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.21, "tpot_shape_error_pct": 6.26, "ttft_mape_pct": 48.39, "ttft_shape_error_pct": 60.23}, "framework": "sglang", "id": "e476e2e8a9cc8057", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12844.105791100716, "interactivity_tok_s": 84.07756190662461, "output_per_gpu": 77.88978551786752, "total_per_gpu": null, "tpot_error_pct": 10.05, "tpot_ms": 11.893779711530959, "tpot_relative": 0.899457, "ttft_error_pct": 26.75, "ttft_ms": 676.7691462045436, "ttft_relative": 1.267475, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11730.568520024994, "interactivity_tok_s": 80.6724432376931, "output_per_gpu": 76.74192774834309, "total_per_gpu": 689.8641626186129, "tpot_error_pct": 6.26, "tpot_ms": 12.395806546401506, "tpot_relative": 0.937422, "ttft_error_pct": 36.94, "ttft_ms": 336.70780645, "ttft_relative": 0.630597, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 12686.97157259976, "interactivity_tok_s": 75.62414856966673, "output_per_gpu": 70.8056994740846, "total_per_gpu": 636.5010107186051, "tpot_ms": 13.223289371367622, "tpot_relative": 1.0, "ttft_ms": 533.950482374803, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17002.466175169895, "interactivity_tok_s": 62.921553219257106, "output_per_gpu": 118.8241451655911, "total_per_gpu": null, "tpot_error_pct": 0.04, "tpot_ms": 15.8928053876132, "tpot_relative": 1.20188, "ttft_error_pct": 13.28, "ttft_ms": 744.1262636415937, "ttft_relative": 1.393624, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15489.92547520014, "interactivity_tok_s": 61.40077139388643, "output_per_gpu": 117.86776461297093, "total_per_gpu": 1048.4882611168648, "tpot_error_pct": 2.51, "tpot_ms": 16.28644033777674, "tpot_relative": 1.231648, "ttft_error_pct": 38.31, "ttft_ms": 405.2081009875009, "ttft_relative": 0.758887, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 15368.823224588232, "interactivity_tok_s": 62.944914177332315, "output_per_gpu": 118.56454992709291, "total_per_gpu": 1054.6864886371238, "tpot_ms": 15.886907037203006, "tpot_relative": 1.201434, "ttft_ms": 656.8823781753053, "ttft_relative": 1.230231, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21976.055021380453, "interactivity_tok_s": 48.48843979160388, "output_per_gpu": 185.57147832375514, "total_per_gpu": null, "tpot_error_pct": 1.44, "tpot_ms": 20.623472404924794, "tpot_relative": 1.559633, "ttft_error_pct": 5.74, "ttft_ms": 878.2427511423869, "ttft_relative": 1.644802, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19723.3673274125, "interactivity_tok_s": 47.689407364849835, "output_per_gpu": 180.95791532752395, "total_per_gpu": 1633.0998925933861, "tpot_error_pct": 0.21, "tpot_ms": 20.969017130983776, "tpot_relative": 1.585764, "ttft_error_pct": 42.68, "ttft_ms": 534.0607389062494, "ttft_relative": 1.000206, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 20087.425073906616, "interactivity_tok_s": 47.79158740092914, "output_per_gpu": 177.22613027848013, "total_per_gpu": 1599.4214665806605, "tpot_ms": 20.924184660594857, "tpot_relative": 1.582374, "ttft_ms": 931.7613286501455, "ttft_relative": 1.745033, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31174.75471401655, "interactivity_tok_s": 34.07216848347359, "output_per_gpu": 264.28966800204563, "total_per_gpu": null, "tpot_error_pct": 0.95, "tpot_ms": 29.349467454208007, "tpot_relative": 2.219528, "ttft_error_pct": 15.13, "ttft_ms": 1150.2495083617616, "ttft_relative": 2.154225, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27827.237011712896, "interactivity_tok_s": 34.22809215874046, "output_per_gpu": 260.6441119233851, "total_per_gpu": 2330.7626397249574, "tpot_error_pct": 0.49, "tpot_ms": 29.21576801190892, "tpot_relative": 2.209418, "ttft_error_pct": 42.71, "ttft_ms": 776.4468312937508, "ttft_relative": 1.454155, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 28282.32876814036, "interactivity_tok_s": 34.395151760822266, "output_per_gpu": 255.78629877061036, "total_per_gpu": 2287.322527751202, "tpot_ms": 29.073865030566548, "tpot_relative": 2.198686, "ttft_ms": 1355.375332249696, "ttft_relative": 2.538391, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 46629.32532064604, "interactivity_tok_s": 22.58815017573951, "output_per_gpu": 354.2415469544131, "total_per_gpu": null, "tpot_error_pct": 0.31, "tpot_ms": 44.271000158040216, "tpot_relative": 3.347957, "ttft_error_pct": 38.17, "ttft_ms": 1340.092158970904, "ttft_relative": 2.509769, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42068.437461761925, "interactivity_tok_s": 22.607969484727832, "output_per_gpu": 345.5245465366969, "total_per_gpu": 3114.481547711897, "tpot_error_pct": 0.22, "tpot_ms": 44.232189922032646, "tpot_relative": 3.345022, "ttft_error_pct": 41.68, "ttft_ms": 1264.163579853122, "ttft_relative": 2.367567, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 42893.20730920792, "interactivity_tok_s": 22.657749044341028, "output_per_gpu": 338.5414995572326, "total_per_gpu": 3051.5379126435932, "tpot_ms": 44.135010854035336, "tpot_relative": 3.337673, "ttft_ms": 2167.469331979555, "ttft_relative": 4.059308, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 70975.83086214717, "interactivity_tok_s": 14.948058411162048, "output_per_gpu": 406.8055514020374, "total_per_gpu": 3672.673813126022, "tpot_error_pct": 3.11, "tpot_ms": 66.89832033659152, "tpot_relative": 5.059129, "ttft_error_pct": 125.74, "ttft_ms": 9340.688006731894, "ttft_relative": 17.493547, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 67772.0732792571, "interactivity_tok_s": 14.483688119351761, "output_per_gpu": 425.809047351985, "total_per_gpu": 3844.238929906455, "tpot_ms": 69.04318787863795, "tpot_relative": 5.221332, "ttft_ms": 4137.776269438405, "ttft_relative": 7.749363, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 139270.52768988415, "interactivity_tok_s": 14.421444746693293, "output_per_gpu": 412.08084208633477, "total_per_gpu": 3707.6612628125995, "tpot_error_pct": 2.68, "tpot_ms": 69.34118027455543, "tpot_relative": 5.243868, "ttft_error_pct": 10.64, "ttft_ms": 75261.47577167323, "ttft_relative": 140.952164, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 133867.61465923558, "interactivity_tok_s": 14.034381801179565, "output_per_gpu": 429.40091416977305, "total_per_gpu": 3863.49709349029, "tpot_ms": 71.25358381770344, "tpot_relative": 5.388492, "ttft_ms": 68025.15871943683, "ttft_relative": 127.399751, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.0, "tpot_shape_error_pct": 14.72, "ttft_mape_pct": 33.86, "ttft_shape_error_pct": 46.94}, "aisimulate": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.58, "tpot_shape_error_pct": 10.48, "ttft_mape_pct": 35.49, "ttft_shape_error_pct": 6.73}, "gpu": "b300", "precisions": ["fp4"], "rows": 8, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.25, "tpot_shape_error_pct": 14.72, "ttft_mape_pct": 33.67, "ttft_shape_error_pct": 46.94}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.19, "tpot_shape_error_pct": 10.48, "ttft_mape_pct": 35.94, "ttft_shape_error_pct": 6.73}, "framework": "sglang", "id": "9d9566ec13f44bdb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 12750.664796294277, "interactivity_tok_s": 84.56431070547634, "output_per_gpu": 78.51238107474605, "total_per_gpu": null, "tpot_error_pct": 11.01, "tpot_ms": 11.825319589996262, "tpot_relative": 0.889882, "ttft_error_pct": 36.22, "ttft_ms": 653.362855728101, "ttft_relative": 1.362152, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11637.153260675048, "interactivity_tok_s": 81.2551683279036, "output_per_gpu": 77.39239897124499, "total_per_gpu": 695.7115109803358, "tpot_error_pct": 7.39, "tpot_ms": 12.306909462848198, "tpot_relative": 0.926123, "ttft_error_pct": 32.22, "ttft_ms": 325.1023318500003, "ttft_relative": 0.677784, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 12692.30711539276, "interactivity_tok_s": 75.25224243380366, "output_per_gpu": 70.77053883902741, "total_per_gpu": 636.1849375787641, "tpot_ms": 13.288640546222386, "tpot_relative": 1.0, "ttft_ms": 479.6549115329981, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16505.105079871468, "interactivity_tok_s": 64.80155537499446, "output_per_gpu": 122.41437077367905, "total_per_gpu": null, "tpot_error_pct": 3.0, "tpot_ms": 15.431728362277529, "tpot_relative": 1.161272, "ttft_error_pct": 16.57, "ttft_ms": 718.4469652615553, "ttft_relative": 1.497841, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15049.348116412508, "interactivity_tok_s": 63.18429324691488, "output_per_gpu": 121.29652394020958, "total_per_gpu": 1078.988660582395, "tpot_error_pct": 0.52, "tpot_ms": 15.826718138511856, "tpot_relative": 1.190996, "ttft_error_pct": 36.56, "ttft_ms": 390.99745368750064, "ttft_relative": 0.815164, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 15347.194510058034, "interactivity_tok_s": 62.857413386420376, "output_per_gpu": 118.7142446039478, "total_per_gpu": 1056.0180919974607, "tpot_ms": 15.909022438649036, "tpot_relative": 1.19719, "ttft_ms": 616.3437319220975, "ttft_relative": 1.284973, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22073.421095959544, "interactivity_tok_s": 48.19752163503105, "output_per_gpu": 184.790796497031, "total_per_gpu": null, "tpot_error_pct": 0.1, "tpot_ms": 20.747954792621066, "tpot_relative": 1.56133, "ttft_error_pct": 1.9, "ttft_ms": 848.2633431081925, "ttft_relative": 1.768487, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19819.761676018687, "interactivity_tok_s": 47.402886944568735, "output_per_gpu": 180.1598355367732, "total_per_gpu": 1625.8974222389004, "tpot_error_pct": 1.57, "tpot_ms": 21.095761554973745, "tpot_relative": 1.587503, "ttft_error_pct": 40.42, "ttft_ms": 515.2203554812498, "ttft_relative": 1.074148, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 19879.87660000217, "interactivity_tok_s": 48.147587314909735, "output_per_gpu": 179.03050526811347, "total_per_gpu": 1615.7054992323133, "tpot_ms": 20.76947269360541, "tpot_relative": 1.562949, "ttft_ms": 864.7339768242091, "ttft_relative": 1.802825, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30700.451518269416, "interactivity_tok_s": 34.573514235225026, "output_per_gpu": 268.3485624337484, "total_per_gpu": null, "tpot_error_pct": 1.65, "tpot_ms": 28.923874882847624, "tpot_relative": 2.176586, "ttft_error_pct": 9.91, "ttft_ms": 1111.3275131162968, "ttft_relative": 2.316931, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27394.04546434367, "interactivity_tok_s": 34.75153568366855, "output_per_gpu": 264.7188983755663, "total_per_gpu": 2367.2006776208336, "tpot_error_pct": 1.13, "tpot_ms": 28.775706751570954, "tpot_relative": 2.165436, "ttft_error_pct": 39.09, "ttft_ms": 751.3328141656255, "ttft_relative": 1.566403, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 27579.959665977003, "interactivity_tok_s": 35.14385683885733, "output_per_gpu": 262.32762892997806, "total_per_gpu": 2345.8171848414872, "tpot_ms": 28.454475118801845, "tpot_relative": 2.141263, "ttft_ms": 1233.5942445835826, "ttft_relative": 2.571837, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 46057.99741543431, "interactivity_tok_s": 22.853438487166127, "output_per_gpu": 358.57888111601153, "total_per_gpu": null, "tpot_error_pct": 2.58, "tpot_ms": 43.757091544958236, "tpot_relative": 3.292819, "ttft_error_pct": 35.21, "ttft_ms": 1294.4927649420372, "ttft_relative": 2.6988, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41596.641310181105, "interactivity_tok_s": 22.84808430372667, "output_per_gpu": 349.4085727693744, "total_per_gpu": 3149.491297825897, "tpot_error_pct": 2.6, "tpot_ms": 43.76734551162758, "tpot_relative": 3.293591, "ttft_error_pct": 38.79, "ttft_ms": 1223.0041247484357, "ttft_relative": 2.549758, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 41355.706836752324, "interactivity_tok_s": 23.442472191799624, "output_per_gpu": 350.90377175312443, "total_per_gpu": 3162.9686894952397, "tpot_ms": 42.65761698758924, "tpot_relative": 3.210081, "ttft_ms": 1998.0025841956376, "ttft_relative": 4.1655, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 72905.5760846146, "interactivity_tok_s": 14.290589652153592, "output_per_gpu": 451.9323365417329, "total_per_gpu": null, "tpot_error_pct": 5.78, "tpot_ms": 69.97611885450087, "tpot_relative": 5.26586, "ttft_error_pct": 58.96, "ttft_ms": 1320.0064964602043, "ttft_relative": 2.751992, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 66324.66483151053, "interactivity_tok_s": 14.355420945911392, "output_per_gpu": 438.6902719013684, "total_per_gpu": 3960.531679404268, "tpot_error_pct": 5.3, "tpot_ms": 69.66009591553028, "tpot_relative": 5.242078, "ttft_error_pct": 32.94, "ttft_ms": 2156.70482242032, "ttft_relative": 4.496368, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 64163.81838786012, "interactivity_tok_s": 15.116315726778158, "output_per_gpu": 453.0002868990765, "total_per_gpu": 4089.723666009148, "tpot_ms": 66.15368573100959, "tpot_relative": 4.978213, "ttft_ms": 3216.2308512671643, "ttft_relative": 6.705302, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 123865.76773270797, "interactivity_tok_s": 8.350973459520715, "output_per_gpu": 531.0621517608174, "total_per_gpu": null, "tpot_error_pct": 5.65, "tpot_ms": 119.7465187558377, "tpot_relative": 9.011194, "ttft_error_pct": 76.92, "ttft_ms": 1365.0790454860141, "ttft_relative": 2.845961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 112771.01491615284, "interactivity_tok_s": 8.495985684185916, "output_per_gpu": 518.7944643089354, "total_per_gpu": 4667.807726613167, "tpot_error_pct": 3.84, "tpot_ms": 117.70264654063149, "tpot_relative": 8.857388, "ttft_error_pct": 31.56, "ttft_ms": 4048.136971939057, "ttft_relative": 8.439686, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 110643.00167068832, "interactivity_tok_s": 8.822635503268325, "output_per_gpu": 528.7125506397188, "total_per_gpu": 4757.044839175105, "tpot_ms": 113.34481625468402, "tpot_relative": 8.529452, "ttft_ms": 5915.201229853119, "ttft_relative": 12.332202, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.18, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.27, "tpot_shape_error_pct": null, "ttft_mape_pct": 32.3, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "9e1e2d87b2fce396", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10776.697769269642, "interactivity_tok_s": 100.59005366718736, "output_per_gpu": 46.340035843059965, "total_per_gpu": null, "tpot_error_pct": 18.23, "tpot_ms": 9.941340754311593, "tpot_relative": 0.817688, "ttft_error_pct": 35.18, "ttft_ms": 606.7061776088834, "ttft_relative": 1.351771, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9884.026088550008, "interactivity_tok_s": 95.94665795865444, "output_per_gpu": 45.47385190872653, "total_per_gpu": 408.78280867441157, "tpot_error_pct": 14.27, "tpot_ms": 10.422457866442022, "tpot_relative": 0.857261, "ttft_error_pct": 32.3, "ttft_ms": 303.85738587500026, "ttft_relative": 0.677009, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26381101926", "measured": {"e2e_ms": 11621.693851776945, "interactivity_tok_s": 82.25129168557102, "output_per_gpu": 38.629300451344626, "total_per_gpu": 347.2543730696815, "tpot_ms": 12.157863779486707, "tpot_relative": 1.0, "ttft_ms": 448.8233529540594, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 50.0, "points": 5, "status_counts": {"failed": 5, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.63, "tpot_shape_error_pct": 5.07, "ttft_mape_pct": 82.47, "ttft_shape_error_pct": 70.94}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.61, "tpot_shape_error_pct": 2.29, "ttft_mape_pct": 25.48, "ttft_shape_error_pct": 18.48}, "gpu": "gb200", "precisions": ["fp4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.76, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.26, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "d3a58f20d0d709df", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 36411.16191542027, "interactivity_tok_s": 28.707235287190965, "output_per_gpu": 769.9225553839464, "total_per_gpu": 6923.6185138885985, "tpot_error_pct": 19.76, "tpot_ms": 34.834423795808554, "tpot_relative": 1.197596, "ttft_error_pct": 22.26, "ttft_ms": 4313.698327793229, "ttft_relative": 1.222606, "unavailable_metrics": {}}, "concurrency": 1127, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 30329.875225702315, "interactivity_tok_s": 34.379660063457685, "output_per_gpu": 2081.0215472428126, "total_per_gpu": 8317.593497424103, "tpot_ms": 29.08696590234483, "tpot_relative": 1.0, "ttft_ms": 3528.281439462516, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.63, "tpot_shape_error_pct": 5.07, "ttft_mape_pct": 82.47, "ttft_shape_error_pct": 70.94}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.24, "tpot_shape_error_pct": 2.29, "ttft_mape_pct": 19.54, "ttft_shape_error_pct": 18.48}, "framework": "dynamo-trt", "id": "42fc0ae6bfe4357e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 15569.746, "interactivity_tok_s": 71.6229766509096, "output_per_gpu": 13.738666666666667, "total_per_gpu": null, "tpot_error_pct": 20.21, "tpot_ms": 13.962, "tpot_relative": 1.202082, "ttft_error_pct": 133.75, "ttft_ms": 1286.62, "ttft_relative": 2.337521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12643.940062662503, "interactivity_tok_s": 77.90658516176877, "output_per_gpu": 14.950790904501929, "total_per_gpu": 132.9941974318058, "tpot_error_pct": 10.51, "tpot_ms": 12.835885412299287, "tpot_relative": 1.105127, "ttft_error_pct": 40.28, "ttft_ms": 772.1417671875007, "ttft_relative": 1.402821, "unavailable_metrics": {}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 11292.347188311396, "interactivity_tok_s": 86.09668933436235, "output_per_gpu": 20.073182176908446, "total_per_gpu": 148.81826383654922, "tpot_ms": 11.614848465501757, "tpot_relative": 1.0, "ttft_ms": 550.420627288986, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16479.193, "interactivity_tok_s": 67.33553296074338, "output_per_gpu": 25.907208333333333, "total_per_gpu": null, "tpot_error_pct": 14.08, "tpot_ms": 14.851, "tpot_relative": 1.278622, "ttft_error_pct": 75.17, "ttft_ms": 1286.62, "ttft_relative": 2.337521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13367.658297306261, "interactivity_tok_s": 73.31122405298454, "output_per_gpu": 27.94267521968892, "total_per_gpu": 252.17564988772267, "tpot_error_pct": 4.78, "tpot_ms": 13.640476106049814, "tpot_relative": 1.1744, "ttft_error_pct": 22.28, "ttft_ms": 898.1344718062522, "ttft_relative": 1.631724, "unavailable_metrics": {}}, "concurrency": 10, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 12634.284242111607, "interactivity_tok_s": 76.81347785103587, "output_per_gpu": 35.62394906697518, "total_per_gpu": 267.9467917696629, "tpot_ms": 13.018548671097756, "tpot_relative": 1.120854, "ttft_ms": 734.5145682251314, "ttft_relative": 1.33446, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19191.166, "interactivity_tok_s": 57.136327276882646, "output_per_gpu": 54.75841666666667, "total_per_gpu": null, "tpot_error_pct": 25.63, "tpot_ms": 17.502, "tpot_relative": 1.506864, "ttft_error_pct": 61.45, "ttft_ms": 1286.62, "ttft_relative": 2.337521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18118.23334606003, "interactivity_tok_s": 64.23836850263511, "output_per_gpu": 51.83964425133448, "total_per_gpu": 464.0643623201167, "tpot_error_pct": 11.74, "tpot_ms": 15.567020509852757, "tpot_relative": 1.340269, "ttft_error_pct": 11.43, "ttft_ms": 3719.3647394250042, "ttft_relative": 6.757314, "unavailable_metrics": {}}, "concurrency": 25, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 16223.260035174899, "interactivity_tok_s": 71.7786005143075, "output_per_gpu": 69.48095457196975, "total_per_gpu": 518.3858387865333, "tpot_ms": 13.931728855603305, "tpot_relative": 1.199476, "ttft_ms": 3337.7960007078946, "ttft_relative": 6.064082, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21634.09, "interactivity_tok_s": 50.27652086475616, "output_per_gpu": 96.36691666666667, "total_per_gpu": null, "tpot_error_pct": 26.86, "tpot_ms": 19.89, "tpot_relative": 1.712463, "ttft_error_pct": 65.92, "ttft_ms": 1286.62, "ttft_relative": 2.337521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20544.526093293778, "interactivity_tok_s": 56.44449539570974, "output_per_gpu": 90.96332956908384, "total_per_gpu": 821.4580100767836, "tpot_error_pct": 13.0, "tpot_ms": 17.716519440724923, "tpot_relative": 1.525334, "ttft_error_pct": 12.52, "ttft_ms": 4247.996646970003, "ttft_relative": 7.717728, "unavailable_metrics": {}}, "concurrency": 50, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 18198.06373315747, "interactivity_tok_s": 63.77948822549453, "output_per_gpu": 122.48684450363592, "total_per_gpu": 921.8907148284605, "tpot_ms": 15.679022015110348, "tpot_relative": 1.349912, "ttft_ms": 3775.4471294931136, "ttft_relative": 6.859204, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24408.466, "interactivity_tok_s": 44.24387222369702, "output_per_gpu": 178.21012499999998, "total_per_gpu": null, "tpot_error_pct": 26.4, "tpot_ms": 22.602, "tpot_relative": 1.945957, "ttft_error_pct": 76.08, "ttft_ms": 1286.62, "ttft_relative": 2.337521, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24242.42682428853, "interactivity_tok_s": 50.29936200740482, "output_per_gpu": 160.85819558017434, "total_per_gpu": 1451.6495711663308, "tpot_error_pct": 11.19, "tpot_ms": 19.880967870979855, "tpot_relative": 1.711686, "ttft_error_pct": 11.19, "ttft_ms": 5979.509328276171, "ttft_relative": 10.863527, "unavailable_metrics": {}}, "concurrency": 105, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29287890247", "measured": {"e2e_ms": 21803.7816564328, "interactivity_tok_s": 55.92635921889027, "output_per_gpu": 213.54907540235598, "total_per_gpu": 1606.1546976050224, "tpot_ms": 17.880656169411964, "tpot_relative": 1.539465, "ttft_ms": 5377.908168881965, "ttft_relative": 9.770543, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.02, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "3f13f384500df8b8", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 30685.67359401304, "interactivity_tok_s": 36.81651261241508, "output_per_gpu": 223.1125262599503, "total_per_gpu": 2008.5574344160154, "tpot_error_pct": 33.33, "tpot_ms": 27.16172524343832, "tpot_relative": 1.333339, "ttft_error_pct": 33.02, "ttft_ms": 5673.180864387184, "ttft_relative": 1.330226, "unavailable_metrics": {}}, "concurrency": 308, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 23025.113485714224, "interactivity_tok_s": 49.0888846333708, "output_per_gpu": 372.78618196213836, "total_per_gpu": 2684.744386419028, "tpot_ms": 20.3712104576969, "tpot_relative": 1.0, "ttft_ms": 4264.825729184743, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.91, "tpot_shape_error_pct": null, "ttft_mape_pct": 24.86, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "d89a4eb4f141eb35", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42870.044927499745, "interactivity_tok_s": 23.21228655949994, "output_per_gpu": 865.288790085036, "total_per_gpu": 7790.866998635452, "tpot_error_pct": 27.91, "tpot_ms": 43.08063307062425, "tpot_relative": 1.279083, "ttft_error_pct": 24.86, "ttft_ms": 3220.9645025993727, "ttft_relative": 1.248553, "unavailable_metrics": {}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 33577.17188973248, "interactivity_tok_s": 29.69043927239808, "output_per_gpu": 3588.415030281997, "total_per_gpu": 9941.960926524738, "tpot_ms": 33.68087588147127, "tpot_relative": 1.0, "ttft_ms": 2579.758126767614, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 52}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.48, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.05, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "f413aceca7872202", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 32651.492033218678, "interactivity_tok_s": 34.45125507111906, "output_per_gpu": 348.63147986136636, "total_per_gpu": 3137.1950431362616, "tpot_error_pct": 38.48, "tpot_ms": 29.026518712762755, "tpot_relative": 1.384779, "ttft_error_pct": 36.05, "ttft_ms": 5919.197841852885, "ttft_relative": 1.360511, "unavailable_metrics": {}}, "concurrency": 615, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 23654.00906196446, "interactivity_tok_s": 47.707388233485275, "output_per_gpu": 724.0677715301655, "total_per_gpu": 4344.070707462074, "tpot_ms": 20.96111392864117, "tpot_relative": 1.0, "ttft_ms": 4350.718008156306, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 40.96, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "84bedc5f0dcb078d", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 38420.27314398377, "interactivity_tok_s": 29.335717201205153, "output_per_gpu": 509.87824456210427, "total_per_gpu": 4585.306010200147, "tpot_error_pct": 45.39, "tpot_ms": 34.08813880844606, "tpot_relative": 1.45392, "ttft_error_pct": 40.96, "ttft_ms": 7009.5866331744855, "ttft_relative": 1.409612, "unavailable_metrics": {}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29287890247", "measured": {"e2e_ms": 26576.78025125062, "interactivity_tok_s": 42.65178628206859, "output_per_gpu": 1291.4737718257816, "total_per_gpu": 6637.057712603336, "tpot_ms": 23.44567689115553, "tpot_relative": 1.0, "ttft_ms": 4972.706100688689, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}]}, {"aic": {"coverage_pct": 19.05, "points": 4, "status_counts": {"failed": 17, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.05, "tpot_shape_error_pct": 15.11, "ttft_mape_pct": 69.57, "ttft_shape_error_pct": 81.24}, "aisimulate": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.23, "tpot_shape_error_pct": 34.3, "ttft_mape_pct": 49.79, "ttft_shape_error_pct": 67.8}, "gpu": "gb300", "precisions": ["fp4"], "rows": 21, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 78.45, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "426943b819e90e9a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8965.41769170001, "interactivity_tok_s": 115.50138707765046, "output_per_gpu": 16.418123460314394, "total_per_gpu": 148.24148442023002, "tpot_error_pct": 29.68, "tpot_ms": 8.65790468237156, "tpot_relative": 0.703166, "ttft_error_pct": 78.45, "ttft_ms": 1045.3498513250029, "ttft_relative": 0.215516, "unavailable_metrics": {}}, "concurrency": 12, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 16109.509884458443, "interactivity_tok_s": 81.21670221240734, "output_per_gpu": 9.456208339867919, "total_per_gpu": 80.64774066294736, "tpot_ms": 12.312738300856934, "tpot_relative": 1.0, "ttft_ms": 4850.457103633289, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 81.75, "tpot_shape_error_pct": 190.76, "ttft_mape_pct": 55.95, "ttft_shape_error_pct": 52.27}, "framework": "dynamo-sglang", "id": "4ca64f795912b190", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26089.489383587563, "interactivity_tok_s": 52.26352878219046, "output_per_gpu": 271.3401923611181, "total_per_gpu": 2449.681464519624, "tpot_error_pct": 14.29, "tpot_ms": 19.13380177920103, "tpot_relative": 0.857055, "ttft_error_pct": 111.13, "ttft_ms": 8505.39179168286, "ttft_relative": 2.111281, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 24545.103342689876, "interactivity_tok_s": 44.79273655834172, "output_per_gpu": 384.22773619569676, "total_per_gpu": 2601.9427801309716, "tpot_ms": 22.32504814028315, "tpot_relative": 1.0, "ttft_ms": 4028.5465215273434, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 197614.41822178173, "interactivity_tok_s": 17.012487791447874, "output_per_gpu": 286.53838862451477, "total_per_gpu": 2578.3166155110316, "tpot_error_pct": 149.2, "tpot_ms": 58.78035077872015, "tpot_relative": 2.632933, "ttft_error_pct": 0.78, "ttft_ms": 143604.66502889903, "ttft_relative": 35.646768, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26130079416", "measured": {"e2e_ms": 164218.335989138, "interactivity_tok_s": 42.39511113153503, "output_per_gpu": 455.73884983732637, "total_per_gpu": 3075.977081365183, "tpot_ms": 23.587625396178368, "tpot_relative": 1.056554, "ttft_ms": 142500.21555564983, "ttft_relative": 35.372613, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.78, "tpot_shape_error_pct": 11.35, "ttft_mape_pct": 29.17, "ttft_shape_error_pct": 85.22}, "framework": "dynamo-sglang", "id": "0c40daef0c070166", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15261.959190285945, "interactivity_tok_s": 68.5609811693219, "output_per_gpu": 155.5709262472094, "total_per_gpu": 1402.2817887004237, "tpot_error_pct": 10.72, "tpot_ms": 14.585555558639774, "tpot_relative": 0.892835, "ttft_error_pct": 31.53, "ttft_ms": 1834.4977924390646, "ttft_relative": 0.684673, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 17720.12602959082, "interactivity_tok_s": 61.21367112865341, "output_per_gpu": 159.73694543167196, "total_per_gpu": 1200.0055467407753, "tpot_ms": 16.33622002343708, "tpot_relative": 1.0, "ttft_ms": 2679.376552021972, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 195696.22107460492, "interactivity_tok_s": 63.82709302409846, "output_per_gpu": 191.15619725259776, "total_per_gpu": 1720.052946134668, "tpot_error_pct": 20.85, "tpot_ms": 15.6673279734429, "tpot_relative": 0.959055, "ttft_error_pct": 26.81, "ttft_ms": 181269.0809413719, "ttft_relative": 67.653455, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26130079416", "measured": {"e2e_ms": 161166.79399588544, "interactivity_tok_s": 50.51993129061285, "output_per_gpu": 278.37967434566787, "total_per_gpu": 2087.6709899170096, "tpot_ms": 19.794167855208677, "tpot_relative": 1.211674, "ttft_ms": 142941.0637156555, "ttft_relative": 53.348628, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.81, "tpot_shape_error_pct": null, "ttft_mape_pct": 24.95, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "24fa01b413f900f7", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 80116.41869153464, "interactivity_tok_s": 38.17205378740746, "output_per_gpu": 432.2368877949574, "total_per_gpu": 3886.6014154654467, "tpot_error_pct": 14.81, "tpot_ms": 26.197175702657347, "tpot_relative": 1.148127, "ttft_error_pct": 24.95, "ttft_ms": 55976.76041773817, "ttft_relative": 1.249481, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 65824.9666849924, "interactivity_tok_s": 43.82636860533295, "output_per_gpu": 852.2865186611721, "total_per_gpu": 4716.522284283493, "tpot_ms": 22.817313681752687, "tpot_relative": 1.0, "ttft_ms": 44800.02490287519, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 52}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.31, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.97, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "558bb4f1f4378828", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 80509.64117397131, "interactivity_tok_s": 32.46930623427507, "output_per_gpu": 620.7268181289064, "total_per_gpu": 5587.766806916819, "tpot_error_pct": 23.31, "tpot_ms": 30.79831742583972, "tpot_relative": 1.233145, "ttft_error_pct": 20.97, "ttft_ms": 52163.09761264418, "ttft_relative": 1.209742, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 66106.62724214482, "interactivity_tok_s": 40.039349271943664, "output_per_gpu": 1693.6372437167686, "total_per_gpu": 6776.369559897207, "tpot_ms": 24.975430874465264, "tpot_relative": 1.0, "ttft_ms": 43119.20491158786, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.64, "tpot_shape_error_pct": 4.16, "ttft_mape_pct": 21.48, "ttft_shape_error_pct": 51.81}, "framework": "dynamo-sglang", "id": "1818682684a753ca", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10707.611384803116, "interactivity_tok_s": 99.07960349204095, "output_per_gpu": 66.33751980371389, "total_per_gpu": 593.2112244145367, "tpot_error_pct": 26.18, "tpot_ms": 10.092894649909756, "tpot_relative": 0.738206, "ttft_error_pct": 17.09, "ttft_ms": 1376.840544059372, "ttft_relative": 0.829057, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 14298.262350018376, "interactivity_tok_s": 73.14113517930467, "output_per_gpu": 54.828247879124255, "total_per_gpu": 441.315805852753, "tpot_ms": 13.672196877290887, "tpot_relative": 1.0, "ttft_ms": 1660.731015212241, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 195245.60884322302, "interactivity_tok_s": 80.67283697302277, "output_per_gpu": 114.89428714245244, "total_per_gpu": 1033.8365165962837, "tpot_error_pct": 23.11, "tpot_ms": 12.395746046893118, "tpot_relative": 0.906639, "ttft_error_pct": 25.86, "ttft_ms": 183830.6084782943, "ttft_relative": 110.692585, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26130079416", "measured": {"e2e_ms": 160904.7756944888, "interactivity_tok_s": 62.030839787365736, "output_per_gpu": 154.8307569065276, "total_per_gpu": 1254.0230701988212, "tpot_ms": 16.12101340926352, "tpot_relative": 1.179109, "ttft_ms": 146061.10774585552, "ttft_relative": 87.949889, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.81, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.38, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "1c5f5f0dafbd91ec", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 85832.61710633316, "interactivity_tok_s": 35.35643092280629, "output_per_gpu": 524.3601590996653, "total_per_gpu": 4719.493314877688, "tpot_error_pct": 20.81, "tpot_ms": 28.283397783653573, "tpot_relative": 1.208103, "ttft_error_pct": 22.38, "ttft_ms": 59792.840793666786, "ttft_relative": 1.223809, "unavailable_metrics": {}}, "concurrency": 3072, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 70412.76010979655, "interactivity_tok_s": 42.71422687531634, "output_per_gpu": 1193.1432866640632, "total_per_gpu": 5727.624743497044, "tpot_ms": 23.411403486688858, "tpot_relative": 1.0, "ttft_ms": 48857.971645990125, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 60}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.12, "tpot_shape_error_pct": 0.42, "ttft_mape_pct": 23.12, "ttft_shape_error_pct": 58.18}, "framework": "dynamo-sglang", "id": "1755e96ec7c13b1a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9045.195722631257, "interactivity_tok_s": 115.2578612345212, "output_per_gpu": 24.16402753426046, "total_per_gpu": 218.07430031120674, "tpot_error_pct": 30.97, "tpot_ms": 8.67619778199118, "tpot_relative": 0.690254, "ttft_error_pct": 20.53, "ttft_ms": 1113.0505115562507, "ttft_relative": 0.794701, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26605445495", "measured": {"e2e_ms": 12895.332521793716, "interactivity_tok_s": 79.55714841972139, "output_per_gpu": 18.198488691917127, "total_per_gpu": 153.99064775915477, "tpot_ms": 12.569580733641661, "tpot_relative": 1.0, "ttft_ms": 1400.5895308809613, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 195126.77801472787, "interactivity_tok_s": 98.80467914666106, "output_per_gpu": 71.79416205820789, "total_per_gpu": 646.0149434773926, "tpot_error_pct": 31.27, "tpot_ms": 10.120978162538705, "tpot_relative": 0.805196, "ttft_error_pct": 25.71, "ttft_ms": 185806.1164514906, "ttft_relative": 132.662791, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26130079416", "measured": {"e2e_ms": 161364.7407840255, "interactivity_tok_s": 67.91049550488955, "output_per_gpu": 92.62286655910891, "total_per_gpu": 781.43934527378, "tpot_ms": 14.7252643728391, "tpot_relative": 1.1715, "ttft_ms": 147806.85741447206, "ttft_relative": 105.531888, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.37, "tpot_shape_error_pct": null, "ttft_mape_pct": 153.67, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "5eb98e9254c2a21c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 32320.642569251282, "interactivity_tok_s": 42.91464035798913, "output_per_gpu": 219.30300459329933, "total_per_gpu": 1972.9454741729749, "tpot_error_pct": 15.37, "tpot_ms": 23.302071080128176, "tpot_relative": 1.153738, "ttft_error_pct": 153.67, "ttft_ms": 10862.840161598378, "ttft_relative": 2.536705, "unavailable_metrics": {}}, "concurrency": 308, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 22881.903310204696, "interactivity_tok_s": 49.512252510306986, "output_per_gpu": 375.59041411083496, "total_per_gpu": 2847.305265530344, "tpot_ms": 20.197020925109992, "tpot_relative": 1.0, "ttft_ms": 4282.263618606994, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 38}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.53, "tpot_shape_error_pct": null, "ttft_mape_pct": 42.47, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "d8f1fe8063562f4c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 32720.8986649722, "interactivity_tok_s": 34.74005037365214, "output_per_gpu": 377.1769337086389, "total_per_gpu": 3396.116174645761, "tpot_error_pct": 38.53, "tpot_ms": 28.78522020677405, "tpot_relative": 1.385344, "ttft_error_pct": 42.47, "ttft_ms": 6215.273855493486, "ttft_relative": 1.424706, "unavailable_metrics": {}}, "concurrency": 615, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 23497.432812711042, "interactivity_tok_s": 48.126926465124505, "output_per_gpu": 728.6478907331554, "total_per_gpu": 4768.9628703206745, "tpot_ms": 20.778389011080037, "tpot_relative": 1.0, "ttft_ms": 4362.495040822949, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.82, "tpot_shape_error_pct": null, "ttft_mape_pct": 80.48, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "27b8a8066d911819", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 38589.441131294305, "interactivity_tok_s": 26.555592700849648, "output_per_gpu": 726.3295811481374, "total_per_gpu": 6536.616570891459, "tpot_error_pct": 28.82, "tpot_ms": 37.65685109216203, "tpot_relative": 1.288163, "ttft_error_pct": 80.48, "ttft_ms": 3914.969873954093, "ttft_relative": 1.804755, "unavailable_metrics": {}}, "concurrency": 1076, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 29107.767268803484, "interactivity_tok_s": 34.20792420695419, "output_per_gpu": 2071.272482306884, "total_per_gpu": 8765.492148776031, "tpot_ms": 29.232992740223278, "tpot_relative": 1.0, "ttft_ms": 2169.253142385932, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 34}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.02, "tpot_shape_error_pct": null, "ttft_mape_pct": 69.43, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "ea6c414f98927f08", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 46282.83933741217, "interactivity_tok_s": 21.94669259572934, "output_per_gpu": 896.1344589279353, "total_per_gpu": 8058.531729410977, "tpot_error_pct": 34.02, "tpot_ms": 45.564952242261434, "tpot_relative": 1.340172, "ttft_error_pct": 69.43, "ttft_ms": 4301.1271570793515, "ttft_relative": 1.694312, "unavailable_metrics": {}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 33829.12920215613, "interactivity_tok_s": 29.412335948487847, "output_per_gpu": 3558.743952647175, "total_per_gpu": 11145.81009067204, "tpot_ms": 33.99933965637341, "tpot_relative": 1.0, "ttft_ms": 2538.5684330653835, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 46}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 50.31, "tpot_shape_error_pct": null, "ttft_mape_pct": 13.95, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "c53ec529603c3441", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 35961.4326980104, "interactivity_tok_s": 28.94028698924042, "output_per_gpu": 559.5632513075726, "total_per_gpu": 5033.9315301459155, "tpot_error_pct": 50.31, "tpot_ms": 34.553907512105376, "tpot_relative": 1.50315, "ttft_error_pct": 13.95, "ttft_ms": 4136.458066368732, "ttft_relative": 0.860491, "unavailable_metrics": {}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 25989.153342265927, "interactivity_tok_s": 43.50158135005443, "output_per_gpu": 1320.4443365452014, "total_per_gpu": 7037.272508909068, "tpot_ms": 22.98767007923377, "tpot_relative": 1.0, "ttft_ms": 4807.091348168702, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 54}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.05, "tpot_shape_error_pct": 15.11, "ttft_mape_pct": 69.57, "ttft_shape_error_pct": 81.24}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.41, "tpot_shape_error_pct": 11.13, "ttft_mape_pct": 69.85, "ttft_shape_error_pct": 75.71}, "framework": "dynamo-trt", "id": "b7f7c64720bfa4e3", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 17613.022, "interactivity_tok_s": 62.688064192577734, "output_per_gpu": 26.335409090909092, "total_per_gpu": null, "tpot_error_pct": 24.0, "tpot_ms": 15.952, "tpot_relative": 1.239983, "ttft_error_pct": 79.07, "ttft_ms": 1294.126, "ttft_relative": 1.790669, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14309.398694479987, "interactivity_tok_s": 68.24435314484413, "output_per_gpu": 28.36297205158695, "total_per_gpu": 254.7421211769862, "tpot_error_pct": 13.9, "tpot_ms": 14.653227027847507, "tpot_relative": 1.139027, "ttft_error_pct": 23.01, "ttft_ms": 889.0119775400005, "ttft_relative": 1.230117, "unavailable_metrics": {}}, "concurrency": 10, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 12481.98691609723, "interactivity_tok_s": 77.73215356886548, "output_per_gpu": 35.76067046659724, "total_per_gpu": 293.42743266361856, "tpot_ms": 12.864689244896153, "tpot_relative": 1.0, "ttft_ms": 722.7052518224809, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20924.473, "interactivity_tok_s": 52.113189848350615, "output_per_gpu": 54.597818181818184, "total_per_gpu": null, "tpot_error_pct": 38.98, "tpot_ms": 19.189, "tpot_relative": 1.491602, "ttft_error_pct": 61.12, "ttft_ms": 1294.126, "ttft_relative": 1.790669, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17252.05727573599, "interactivity_tok_s": 57.39073943710395, "output_per_gpu": 58.11959775773292, "total_per_gpu": 523.8534617084355, "tpot_error_pct": 26.2, "tpot_ms": 17.424413935212087, "tpot_relative": 1.354437, "ttft_error_pct": 61.82, "ttft_ms": 1270.6686406000006, "ttft_relative": 1.758211, "unavailable_metrics": {}}, "concurrency": 25, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 16098.732790620417, "interactivity_tok_s": 72.4294784360858, "output_per_gpu": 69.88930581938651, "total_per_gpu": 568.8354321645813, "tpot_ms": 13.806533218134845, "tpot_relative": 1.073212, "ttft_ms": 3328.5155261174077, "ttft_relative": 4.605634, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24396.535, "interactivity_tok_s": 44.28109639994687, "output_per_gpu": 92.70922727272728, "total_per_gpu": null, "tpot_error_pct": 45.42, "tpot_ms": 22.583, "tpot_relative": 1.755425, "ttft_error_pct": 66.14, "ttft_ms": 1294.126, "ttft_relative": 1.790669, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20879.416233805903, "interactivity_tok_s": 48.916727938573004, "output_per_gpu": 96.53873517835615, "total_per_gpu": 865.9999291968591, "tpot_error_pct": 31.64, "tpot_ms": 20.44290454700376, "tpot_relative": 1.589071, "ttft_error_pct": 48.24, "ttft_ms": 1978.101820187994, "ttft_relative": 2.73708, "unavailable_metrics": {}}, "concurrency": 50, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 18106.658136856568, "interactivity_tok_s": 64.39453150235674, "output_per_gpu": 122.85750454323727, "total_per_gpu": 1008.7423286020545, "tpot_ms": 15.529268971595851, "tpot_relative": 1.207124, "ttft_ms": 3821.875266204588, "ttft_relative": 5.28829, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 27527.938, "interactivity_tok_s": 38.99547652472314, "output_per_gpu": 116.54050000000001, "total_per_gpu": null, "tpot_error_pct": 43.81, "tpot_ms": 25.644, "tpot_relative": 1.993363, "ttft_error_pct": 71.97, "ttft_ms": 1294.126, "ttft_relative": 1.790669, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31376.596286423348, "interactivity_tok_s": 46.01152729144135, "output_per_gpu": 128.39013597966525, "total_per_gpu": 1156.5163388216124, "tpot_error_pct": 21.88, "tpot_ms": 21.733684119327442, "tpot_relative": 1.689406, "ttft_error_pct": 146.3, "ttft_ms": 11370.8008500941, "ttft_relative": 15.733663, "unavailable_metrics": {}}, "concurrency": 100, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27622347964", "measured": {"e2e_ms": 21003.30621184752, "interactivity_tok_s": 56.079590773531784, "output_per_gpu": 213.09257417724598, "total_per_gpu": 1748.36172515797, "tpot_ms": 17.831799166265956, "tpot_relative": 1.386104, "ttft_ms": 4616.617189275712, "ttft_relative": 6.387967, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 47}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 17, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.33, "tpot_shape_error_pct": 10.39, "ttft_mape_pct": 82.58, "ttft_shape_error_pct": 92.54}, "frameworks": ["dynamo-sglang"], "gpu_skus": ["gb200"], "hf_model_paths": ["nvidia/GLM-5.1-NVFP4"], "model": "nvidia/GLM-5.1-NVFP4", "precisions": ["fp4"], "rows": 17, "workloads": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.11, "tpot_shape_error_pct": 15.81, "ttft_mape_pct": 59.34, "ttft_shape_error_pct": 37.97}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.11, "tpot_shape_error_pct": 15.81, "ttft_mape_pct": 59.34, "ttft_shape_error_pct": 37.97}, "gpu": "gb200", "precisions": ["fp4"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.3, "tpot_shape_error_pct": 14.89, "ttft_mape_pct": 66.17, "ttft_shape_error_pct": 17.16}, "framework": "dynamo-sglang", "id": "870aee4300876554", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 23112.617915980067, "interactivity_tok_s": 41.36460271079197, "output_per_gpu": 588.9975214361676, "total_per_gpu": 1177.7057678916058, "tpot_error_pct": 6.38, "tpot_ms": 24.175259387638246, "tpot_relative": 1.063815, "ttft_error_pct": 68.85, "ttft_ms": 844.1589497713999, "ttft_relative": 0.311527, "unavailable_metrics": {}}, "concurrency": 556, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 23639.371000079056, "interactivity_tok_s": 44.00428775472033, "output_per_gpu": 644.675209515162, "total_per_gpu": 1146.2612952402778, "tpot_ms": 22.725058193737727, "tpot_relative": 1.0, "ttft_ms": 2709.7474416827604, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 28442.651024115938, "interactivity_tok_s": 34.17780270096993, "output_per_gpu": 998.7185656023444, "total_per_gpu": 1997.1932198375466, "tpot_error_pct": 22.22, "tpot_ms": 29.25875629715719, "tpot_relative": 1.287511, "ttft_error_pct": 63.5, "ttft_ms": 1496.2966562608917, "ttft_relative": 0.55219, "unavailable_metrics": {}}, "concurrency": 1156, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 26146.663282609545, "interactivity_tok_s": 41.77288216329914, "output_per_gpu": 1209.6460604895876, "total_per_gpu": 2150.9686421210718, "tpot_ms": 23.93897543604451, "tpot_relative": 1.053418, "ttft_ms": 4099.711557463775, "ttft_relative": 1.51295, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 66.87, "tpot_shape_error_pct": null, "ttft_mape_pct": 86.99, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "d67cce18deaafec7", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41486.48854228909, "interactivity_tok_s": 23.281699733840817, "output_per_gpu": 1692.8192848290055, "total_per_gpu": 3386.1003563346612, "tpot_error_pct": 66.87, "tpot_ms": 42.952190408437524, "tpot_relative": 1.668691, "ttft_error_pct": 86.99, "ttft_ms": 1943.7781584347588, "ttft_relative": 0.130124, "unavailable_metrics": {}}, "concurrency": 3160, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 38635.033863281926, "interactivity_tok_s": 38.84997200462461, "output_per_gpu": 2241.427507174378, "total_per_gpu": 3587.8611484028415, "tpot_ms": 25.740044288344976, "tpot_relative": 1.0, "ttft_ms": 14937.940645328476, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 59.01, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "452c2f7aec0d71ac", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42792.834234283604, "interactivity_tok_s": 28.119676916759325, "output_per_gpu": 2188.4851411262093, "total_per_gpu": 4375.687046901889, "tpot_error_pct": 18.28, "tpot_ms": 35.56228625813265, "tpot_relative": 1.182761, "ttft_error_pct": 59.01, "ttft_ms": 10026.1154260295, "ttft_relative": 1.590062, "unavailable_metrics": {}}, "concurrency": 2115, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 34009.62781701836, "interactivity_tok_s": 33.25886771437634, "output_per_gpu": 3420.6381918135594, "total_per_gpu": 5472.997401711943, "tpot_ms": 30.067169110743478, "tpot_relative": 1.0, "ttft_ms": 6305.4876422066345, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.35, "tpot_shape_error_pct": 16.28, "ttft_mape_pct": 45.69, "ttft_shape_error_pct": 48.37}, "framework": "dynamo-sglang", "id": "bdcea1e153a12426", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10879.809855730451, "interactivity_tok_s": 120.78316321576304, "output_per_gpu": 26.787487307376242, "total_per_gpu": 53.67753629177577, "tpot_error_pct": 26.34, "tpot_ms": 8.279299642232694, "tpot_relative": 0.736583, "ttft_error_pct": 58.93, "ttft_ms": 3280.6758714434814, "ttft_relative": 0.410689, "unavailable_metrics": {}}, "concurrency": 23, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 18301.33861938451, "interactivity_tok_s": 88.96681922334646, "output_per_gpu": 16.419480031585046, "total_per_gpu": 30.96906372864115, "tpot_ms": 11.240145581574106, "tpot_relative": 1.0, "ttft_ms": 7988.222117468958, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11002.19230139042, "interactivity_tok_s": 96.51452472641306, "output_per_gpu": 85.01547519910146, "total_per_gpu": 170.39064488476853, "tpot_error_pct": 18.34, "tpot_ms": 10.361134791210661, "tpot_relative": 0.921797, "ttft_error_pct": 53.99, "ttft_ms": 1478.6527531136987, "ttft_relative": 0.185104, "unavailable_metrics": {}}, "concurrency": 73, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 14875.137411667136, "interactivity_tok_s": 78.81723513205945, "output_per_gpu": 66.26999115256658, "total_per_gpu": 125.04918747640407, "tpot_ms": 12.68758030301983, "tpot_relative": 1.128774, "ttft_ms": 3213.592047257706, "ttft_relative": 0.402291, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15876.406760425181, "interactivity_tok_s": 66.69485894870982, "output_per_gpu": 236.21178042360128, "total_per_gpu": 472.330744217817, "tpot_error_pct": 10.37, "tpot_ms": 14.993659417872484, "tpot_relative": 1.333938, "ttft_error_pct": 24.14, "ttft_ms": 2074.7548787534456, "ttft_relative": 0.259727, "unavailable_metrics": {}}, "concurrency": 290, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 18131.711220309815, "interactivity_tok_s": 59.779038755461656, "output_per_gpu": 218.76301224327196, "total_per_gpu": 411.8570870610778, "tpot_ms": 16.728271662090517, "tpot_relative": 1.488261, "ttft_ms": 2735.1155176368575, "ttft_relative": 0.342394, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 68}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 7}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.68, "tpot_shape_error_pct": 4.96, "ttft_mape_pct": 98.84, "ttft_shape_error_pct": 147.12}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.68, "tpot_shape_error_pct": 4.96, "ttft_mape_pct": 98.84, "ttft_shape_error_pct": 147.12}, "gpu": "gb200", "precisions": ["fp4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.88, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.22, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "878fa525675ce5ba", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 58234.69833931847, "interactivity_tok_s": 29.98168348045919, "output_per_gpu": 562.0148827592653, "total_per_gpu": 5055.63920145571, "tpot_error_pct": 29.88, "tpot_ms": 33.353697455039786, "tpot_relative": 1.298754, "ttft_error_pct": 31.22, "ttft_ms": 27513.267993170804, "ttft_relative": 1.312226, "unavailable_metrics": {}}, "concurrency": 2674, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 44621.21138714528, "interactivity_tok_s": 38.93881666025243, "output_per_gpu": 1646.2732011373346, "total_per_gpu": 6582.313796027999, "tpot_ms": 25.681314579360855, "tpot_relative": 1.0, "ttft_ms": 20966.865292370316, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.0, "tpot_shape_error_pct": null, "ttft_mape_pct": 151.71, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "b5540941a8329b4e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 28095.48212794577, "interactivity_tok_s": 77.24002760305986, "output_per_gpu": 112.85018480873707, "total_per_gpu": 1018.5917768145074, "tpot_error_pct": 21.0, "tpot_ms": 12.94665513506866, "tpot_relative": 0.78998, "ttft_error_pct": 151.71, "ttft_ms": 16195.747146279811, "ttft_relative": 2.517105, "unavailable_metrics": {}}, "concurrency": 130, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 21496.69590248189, "interactivity_tok_s": 61.01803981652338, "output_per_gpu": 166.37812695501918, "total_per_gpu": 1335.0388133924491, "tpot_ms": 16.388595946492615, "tpot_relative": 1.0, "ttft_ms": 6434.276822437891, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 68.02, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "aeed60f112fd0343", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 53967.52139855377, "interactivity_tok_s": 37.18645872425846, "output_per_gpu": 434.9514311651155, "total_per_gpu": 3912.55069974489, "tpot_error_pct": 14.33, "tpot_ms": 26.891509283395504, "tpot_relative": 1.143325, "ttft_error_pct": 68.02, "ttft_ms": 29198.241570537746, "ttft_relative": 1.680186, "unavailable_metrics": {}}, "concurrency": 1497, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 39042.47364728828, "interactivity_tok_s": 42.516202340666766, "output_per_gpu": 1031.902279590413, "total_per_gpu": 5304.451547649724, "tpot_ms": 23.52044502910599, "tpot_relative": 1.0, "ttft_ms": 17377.98120837258, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.67, "tpot_shape_error_pct": 10.12, "ttft_mape_pct": 133.77, "ttft_shape_error_pct": 57.64}, "framework": "dynamo-sglang", "id": "2972363d144b18ec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22619.2094666747, "interactivity_tok_s": 69.24525884005965, "output_per_gpu": 143.5597151063948, "total_per_gpu": 1293.7889870208357, "tpot_error_pct": 16.44, "tpot_ms": 14.441421936334532, "tpot_relative": 0.835615, "ttft_error_pct": 81.47, "ttft_ms": 9328.10155313105, "ttft_relative": 1.814725, "unavailable_metrics": {}}, "concurrency": 103, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 21044.680818732268, "interactivity_tok_s": 57.86238089256491, "output_per_gpu": 180.01947353755148, "total_per_gpu": 1390.7717554682215, "tpot_ms": 17.28238597469286, "tpot_relative": 1.0, "ttft_ms": 5140.22923905981, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 31515.6433056876, "interactivity_tok_s": 69.07443826973066, "output_per_gpu": 145.63972061025575, "total_per_gpu": 1314.356163917851, "tpot_error_pct": 24.9, "tpot_ms": 14.477135465004764, "tpot_relative": 0.837682, "ttft_error_pct": 186.07, "ttft_ms": 18211.243301009534, "ttft_relative": 3.542885, "unavailable_metrics": {}}, "concurrency": 146, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 24077.52308175829, "interactivity_tok_s": 51.87822523112977, "output_per_gpu": 224.27302231057038, "total_per_gpu": 1735.0651376444657, "tpot_ms": 19.275909990073163, "tpot_relative": 1.11535, "ttft_ms": 6365.999913395523, "ttft_relative": 1.238466, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.94, "tpot_shape_error_pct": 0.59, "ttft_mape_pct": 88.73, "ttft_shape_error_pct": 6.46}, "framework": "dynamo-sglang", "id": "075e8376c3deee1c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 37557.635118903745, "interactivity_tok_s": 43.50379734670587, "output_per_gpu": 331.8532011807943, "total_per_gpu": 2988.58873505878, "tpot_error_pct": 1.24, "tpot_ms": 22.986499133177862, "tpot_relative": 1.012419, "ttft_error_pct": 95.04, "ttft_ms": 16392.990331656853, "ttft_relative": 1.950352, "unavailable_metrics": {}}, "concurrency": 686, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 29311.00078344287, "interactivity_tok_s": 44.04406103735858, "output_per_gpu": 639.7147931548542, "total_per_gpu": 3840.67391670225, "tpot_ms": 22.704536694556634, "tpot_relative": 1.0, "ttft_ms": 8405.145934881417, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 53180.421458359335, "interactivity_tok_s": 43.43364282890259, "output_per_gpu": 339.020767192209, "total_per_gpu": 3050.4527001918473, "tpot_error_pct": 0.65, "tpot_ms": 23.02362719008587, "tpot_relative": 1.014054, "ttft_error_pct": 82.43, "ttft_ms": 31975.654355257742, "ttft_relative": 3.804295, "unavailable_metrics": {}}, "concurrency": 989, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 38597.525470431836, "interactivity_tok_s": 43.71410613045162, "output_per_gpu": 688.1520893467737, "total_per_gpu": 4127.805867735368, "tpot_ms": 22.875910970609816, "tpot_relative": 1.007548, "ttft_ms": 17527.582391501706, "ttft_relative": 2.085339, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 145.7, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "624eb34c1349f94d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 29065.209771381065, "interactivity_tok_s": 58.778113205365415, "output_per_gpu": 201.5682940100207, "total_per_gpu": 1820.2439845287954, "tpot_error_pct": 18.97, "tpot_ms": 17.013135425189482, "tpot_relative": 0.810251, "ttft_error_pct": 145.7, "ttft_ms": 13432.782313002952, "ttft_relative": 2.456994, "unavailable_metrics": {}}, "concurrency": 133, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 24760.028831015476, "interactivity_tok_s": 47.62503233329703, "output_per_gpu": 296.47784857482344, "total_per_gpu": 2142.1106452417503, "tpot_ms": 20.997361072673755, "tpot_relative": 1.0, "ttft_ms": 5467.161581754208, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.69, "tpot_shape_error_pct": 4.17, "ttft_mape_pct": 73.39, "ttft_shape_error_pct": 377.24}, "framework": "dynamo-sglang", "id": "0d9c1a81ecfa3e01", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12130.994432765217, "interactivity_tok_s": 109.76029929500483, "output_per_gpu": 23.98593998968811, "total_per_gpu": 216.2497567435155, "tpot_error_pct": 23.08, "tpot_ms": 9.110762328665679, "tpot_relative": 0.769168, "ttft_error_pct": 61.09, "ttft_ms": 3779.959689530431, "ttft_relative": 0.389061, "unavailable_metrics": {}}, "concurrency": 23, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 20574.827198277268, "interactivity_tok_s": 84.42415891184193, "output_per_gpu": 14.67871085828421, "total_per_gpu": 124.56910809500789, "tpot_ms": 11.844950697634168, "tpot_relative": 1.0, "ttft_ms": 9715.597103548276, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24300.16760088846, "interactivity_tok_s": 98.12167797468638, "output_per_gpu": 59.88188164727212, "total_per_gpu": 541.0553296068583, "tpot_error_pct": 26.29, "tpot_ms": 10.191427833694222, "tpot_relative": 0.860403, "ttft_error_pct": 85.68, "ttft_ms": 14945.26344488938, "ttft_relative": 1.538275, "unavailable_metrics": {}}, "concurrency": 113, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28828457171", "measured": {"e2e_ms": 20739.242820814663, "interactivity_tok_s": 72.32188762532606, "output_per_gpu": 74.47335163323335, "total_per_gpu": 633.3889917345512, "tpot_ms": 13.827072727700967, "tpot_relative": 1.167339, "ttft_ms": 8049.066738296812, "ttft_relative": 0.828469, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 68}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 10}]}, {"aic": {"coverage_pct": 46.28, "points": 87, "status_counts": {"failed": 101, "success": 87, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.76, "tpot_shape_error_pct": 12.43, "ttft_mape_pct": 51.09, "ttft_shape_error_pct": 39.05}, "aisimulate": {"coverage_pct": 62.77, "points": 118, "status_counts": {"failed": 70, "success": 118, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.6, "tpot_shape_error_pct": 11.55, "ttft_mape_pct": 59.06, "ttft_shape_error_pct": 64.72}, "frameworks": ["dynamo-trt", "dynamo-vllm", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300"], "hf_model_paths": ["nvidia/Kimi-K2.5-NVFP4"], "model": "nvidia/Kimi-K2.5-NVFP4", "precisions": ["fp4"], "rows": 188, "workloads": [{"aic": {"coverage_pct": 34.48, "points": 20, "status_counts": {"failed": 38, "success": 20, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 79.91, "tpot_shape_error_pct": 15.05, "ttft_mape_pct": 80.66, "ttft_shape_error_pct": 46.95}, "aisimulate": {"coverage_pct": 46.55, "points": 27, "status_counts": {"failed": 31, "success": 27, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.57, "tpot_shape_error_pct": 15.86, "ttft_mape_pct": 64.17, "ttft_shape_error_pct": 44.67}, "gpu_skus": ["gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 52.78, "points": 19, "status_counts": {"failed": 17, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 83.51, "tpot_shape_error_pct": 15.05, "ttft_mape_pct": 79.78, "ttft_shape_error_pct": 46.95}, "aisimulate": {"coverage_pct": 50.0, "points": 18, "status_counts": {"failed": 18, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.04, "tpot_shape_error_pct": 20.79, "ttft_mape_pct": 70.42, "ttft_shape_error_pct": 42.48}, "gpu": "gb200", "precisions": ["fp4"], "rows": 36, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "0777b43c25b426f8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 9736.482751276344, "interactivity_tok_s": 186.40570551144785, "output_per_gpu": 22.7117808142218, "total_per_gpu": 40.358247559169136, "tpot_ms": 5.364642660782646, "tpot_relative": 1.0, "ttft_ms": 4772.137123561697, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 7858.271462738048, "interactivity_tok_s": 172.81469884261082, "output_per_gpu": 41.75796860477651, "total_per_gpu": 74.69624573326914, "tpot_ms": 5.7865448176415795, "tpot_relative": 1.078645, "ttft_ms": 2558.684198834817, "ttft_relative": 0.536172, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 8650.262268149527, "interactivity_tok_s": 157.24005404325678, "output_per_gpu": 76.65355829714616, "total_per_gpu": 136.6513586011814, "tpot_ms": 6.359702723867672, "tpot_relative": 1.185485, "ttft_ms": 2815.5795414175373, "ttft_relative": 0.590004, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 15195.043525584939, "interactivity_tok_s": 89.97944838764465, "output_per_gpu": 349.2048331280523, "total_per_gpu": 622.1438883235365, "tpot_ms": 11.113648926717726, "tpot_relative": 2.071648, "ttft_ms": 4971.042732996526, "ttft_relative": 1.041681, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "bef482452f5b54b1", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 30, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 9892.821668150911, "interactivity_tok_s": 137.96283375863138, "output_per_gpu": 132.7966162569577, "total_per_gpu": 221.43965435202972, "tpot_ms": 7.248328935817012, "tpot_relative": 1.0, "ttft_ms": 3213.8443079497665, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 12071.988941278929, "interactivity_tok_s": 113.3360323452145, "output_per_gpu": 218.8463779707638, "total_per_gpu": 365.0964473895276, "tpot_ms": 8.823319286086019, "tpot_relative": 1.21729, "ttft_ms": 3939.466985598362, "ttft_relative": 1.22578, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "790f46dfb92ed44a", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 22773.584503421862, "interactivity_tok_s": 52.752539508309454, "output_per_gpu": 1606.1075567135051, "total_per_gpu": 2573.4200041988533, "tpot_ms": 18.956433364548875, "tpot_relative": 1.0, "ttft_ms": 5319.9935237675445, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "14e2781d8d698f9b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 26490.956231075175, "interactivity_tok_s": 38.19846056269591, "output_per_gpu": 8974.049085254368, "total_per_gpu": 11986.157618807192, "tpot_ms": 26.179065471988846, "tpot_relative": 1.0, "ttft_ms": 2396.097923679536, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 41052.8521079354, "interactivity_tok_s": 24.616578261243752, "output_per_gpu": 11135.36289608569, "total_per_gpu": 14868.623965436032, "tpot_ms": 40.62303011358797, "tpot_relative": 1.551737, "ttft_ms": 3658.1793534941826, "ttft_relative": 1.526724, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "f9eae5dd44a6b078", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 333, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 13672.530576948555, "interactivity_tok_s": 87.76835064450547, "output_per_gpu": 667.5538305918244, "total_per_gpu": 1187.3764005869089, "tpot_ms": 11.393628713046832, "tpot_relative": 1.0, "ttft_ms": 3172.677477600134, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 14588.485560103742, "interactivity_tok_s": 82.79402623833911, "output_per_gpu": 1247.4075904587137, "total_per_gpu": 2220.76150176297, "tpot_ms": 12.078166087989255, "tpot_relative": 1.060081, "ttft_ms": 3467.973220810104, "ttft_relative": 1.093075, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1127, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 14130.322527600256, "interactivity_tok_s": 74.47799110643784, "output_per_gpu": 2187.6963236076263, "total_per_gpu": 3894.021506636216, "tpot_ms": 13.426785351539383, "tpot_relative": 1.178447, "ttft_ms": 1764.125274970653, "ttft_relative": 0.556037, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "4cd3ba7f583342aa", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 15749.269185457097, "interactivity_tok_s": 66.96311450657073, "output_per_gpu": 3930.810680431448, "total_per_gpu": 6296.093459964847, "tpot_ms": 14.933594522427349, "tpot_relative": 1.0, "ttft_ms": 1989.9904982616295, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 20898.98928943558, "interactivity_tok_s": 49.02326367744853, "output_per_gpu": 5690.910351568572, "total_per_gpu": 9121.25367094119, "tpot_ms": 20.398478701449974, "tpot_relative": 1.365946, "ttft_ms": 2125.3218773727986, "ttft_relative": 1.068006, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "a031989a091e4a16", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 45819.28585195756, "interactivity_tok_s": 21.81871261973397, "output_per_gpu": 10422.996617671244, "total_per_gpu": 13921.43937846097, "tpot_ms": 45.8322183085884, "tpot_relative": 1.0, "ttft_ms": 3636.7287133444433, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 6452, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 60677.656742354, "interactivity_tok_s": 18.10137019101744, "output_per_gpu": 11834.981245304558, "total_per_gpu": 15802.269159964793, "tpot_ms": 55.24443671652195, "tpot_relative": 1.205362, "ttft_ms": 9822.987941721445, "ttft_relative": 2.701051, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 98.13, "tpot_shape_error_pct": 26.11, "ttft_mape_pct": 91.46, "ttft_shape_error_pct": 77.5}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 60.8, "tpot_shape_error_pct": 44.01, "ttft_mape_pct": 69.58, "ttft_shape_error_pct": 48.91}, "framework": "dynamo-vllm", "id": "94a4a90ed5fa86b3", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 32508.899, "interactivity_tok_s": 31.600568810238585, "output_per_gpu": 372.49845, "total_per_gpu": null, "tpot_error_pct": 66.22, "tpot_ms": 31.645, "tpot_relative": 1.662234, "ttft_error_pct": 75.89, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20899.36591800977, "interactivity_tok_s": 44.646425253235925, "output_per_gpu": 542.471015535453, "total_per_gpu": 1084.417734593922, "tpot_error_pct": 17.65, "tpot_ms": 22.398209808018642, "tpot_relative": 1.176523, "ttft_error_pct": 51.62, "ttft_ms": 272.9756291210948, "ttft_relative": 0.483753, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 18057.830170852685, "interactivity_tok_s": 52.52754139221728, "output_per_gpu": 783.4609632026566, "total_per_gpu": 1256.2354524418542, "tpot_ms": 19.037631944985048, "tpot_relative": 1.0, "ttft_ms": 564.2877826157928, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32078.216, "interactivity_tok_s": 32.026646169613116, "output_per_gpu": 755.03, "total_per_gpu": null, "tpot_error_pct": 55.83, "tpot_ms": 31.224, "tpot_relative": 1.64012, "ttft_error_pct": 83.11, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22988.442155115303, "interactivity_tok_s": 40.84348906748446, "output_per_gpu": 986.0485012876845, "total_per_gpu": 1971.1886753538079, "tpot_error_pct": 22.19, "tpot_ms": 24.483706530255784, "tpot_relative": 1.286069, "ttft_error_pct": 46.34, "ttft_ms": 432.2610285054693, "ttft_relative": 0.766029, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 19234.676445045625, "interactivity_tok_s": 49.908166593875634, "output_per_gpu": 1467.6368207586368, "total_per_gpu": 2352.7251625611866, "tpot_ms": 20.036800953588077, "tpot_relative": 1.052484, "ttft_ms": 805.6028726276054, "ttft_relative": 1.427645, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35150.285, "interactivity_tok_s": 29.216700265871975, "output_per_gpu": 1377.6036, "total_per_gpu": null, "tpot_error_pct": 63.84, "tpot_ms": 34.227, "tpot_relative": 1.79786, "ttft_error_pct": 93.01, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27706.48386212931, "interactivity_tok_s": 34.13880990498389, "output_per_gpu": 1638.8962124961988, "total_per_gpu": 3277.6959031635283, "tpot_error_pct": 40.22, "tpot_ms": 29.2921751749176, "tpot_relative": 1.538646, "ttft_error_pct": 62.43, "ttft_ms": 731.1612872572226, "ttft_relative": 1.295724, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 21155.373612917992, "interactivity_tok_s": 47.869261731509376, "output_per_gpu": 2656.7940910441193, "total_per_gpu": 4259.834003248688, "tpot_ms": 20.890232350121284, "tpot_relative": 1.097313, "ttft_ms": 1946.367306898719, "ttft_relative": 3.449246, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45454.964, "interactivity_tok_s": 22.573363431151243, "output_per_gpu": 2128.67685, "total_per_gpu": null, "tpot_error_pct": 97.11, "tpot_ms": 44.3, "tpot_relative": 2.32697, "ttft_error_pct": 98.37, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35180.57087705289, "interactivity_tok_s": 27.19302356691203, "output_per_gpu": 2587.832414776275, "total_per_gpu": 5174.530176709089, "tpot_error_pct": 63.63, "tpot_ms": 36.77413795267628, "tpot_relative": 1.931655, "ttft_error_pct": 84.47, "ttft_ms": 1299.8883451886684, "ttft_relative": 2.303591, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 29048.758277165325, "interactivity_tok_s": 44.49490968652086, "output_per_gpu": 3785.016076709902, "total_per_gpu": 6065.684606447591, "tpot_ms": 22.474480947265224, "tpot_relative": 1.180529, "ttft_ms": 8368.63005977707, "ttft_relative": 14.830429, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 55231.775, "interactivity_tok_s": 18.567688508457582, "output_per_gpu": 2438.3693, "total_per_gpu": null, "tpot_error_pct": 132.49, "tpot_ms": 53.857, "tpot_relative": 2.828976, "ttft_error_pct": 99.06, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43972.71513384474, "interactivity_tok_s": 21.885342776925448, "output_per_gpu": 3111.1481430963922, "total_per_gpu": 6223.206083788356, "tpot_error_pct": 97.25, "tpot_ms": 45.69268163596406, "tpot_relative": 2.400124, "ttft_error_pct": 86.84, "ttft_ms": 1905.4528662918894, "ttft_relative": 3.37674, "unavailable_metrics": {}}, "concurrency": 3072, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 35778.478695017715, "interactivity_tok_s": 43.168847693070006, "output_per_gpu": 4446.300276724905, "total_per_gpu": 7129.528649536985, "tpot_ms": 23.16485274543319, "tpot_relative": 1.216793, "ttft_ms": 14481.143914133949, "ttft_relative": 25.662693, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 66467.384, "interactivity_tok_s": 15.42257865515114, "output_per_gpu": 2438.3693, "total_per_gpu": null, "tpot_error_pct": 173.29, "tpot_ms": 64.84, "tpot_relative": 3.405886, "ttft_error_pct": 99.32, "ttft_ms": 136.064, "ttft_relative": 0.241125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51723.37317613376, "interactivity_tok_s": 18.828028971992104, "output_per_gpu": 3529.8583576290316, "total_per_gpu": 7062.281836664207, "tpot_error_pct": 123.86, "tpot_ms": 53.1123040806642, "tpot_relative": 2.789859, "ttft_error_pct": 85.77, "ttft_ms": 2839.903572665997, "ttft_relative": 5.032722, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 41763.45501116707, "interactivity_tok_s": 42.148945421046534, "output_per_gpu": 4974.527375058007, "total_per_gpu": 7978.603680832699, "tpot_ms": 23.725386009317873, "tpot_relative": 1.246236, "ttft_ms": 19955.467698171266, "ttft_relative": 35.36399, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.0, "tpot_shape_error_pct": 6.76, "ttft_mape_pct": 59.62, "ttft_shape_error_pct": 31.95}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.58, "tpot_shape_error_pct": 5.93, "ttft_mape_pct": 67.09, "ttft_shape_error_pct": 11.1}, "framework": "dynamo-vllm", "id": "20dbf7265f305ad7", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9608.021, "interactivity_tok_s": 108.00302408467437, "output_per_gpu": 19.9721, "total_per_gpu": null, "tpot_error_pct": 34.44, "tpot_ms": 9.259, "tpot_relative": 1.344353, "ttft_error_pct": 44.97, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7950.456030000001, "interactivity_tok_s": 116.40022455713627, "output_per_gpu": 22.322828336928925, "total_per_gpu": 44.86396540939838, "tpot_error_pct": 24.74, "tpot_ms": 8.591048718373731, "tpot_relative": 1.24737, "ttft_error_pct": 69.8, "ttft_ms": 74.66851325000145, "ttft_relative": 0.301973, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 6561.801320291124, "interactivity_tok_s": 145.19416933817232, "output_per_gpu": 33.37177126905024, "total_per_gpu": 53.80137891352049, "tpot_ms": 6.8873289096816, "tpot_relative": 1.0, "ttft_ms": 247.2690472844988, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10390.616, "interactivity_tok_s": 99.76057462090982, "output_per_gpu": 36.7411, "total_per_gpu": null, "tpot_error_pct": 34.43, "tpot_ms": 10.024, "tpot_relative": 1.455426, "ttft_error_pct": 51.42, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8651.46877388751, "interactivity_tok_s": 108.33771208076892, "output_per_gpu": 41.96295713501239, "total_per_gpu": 83.61595996780049, "tpot_error_pct": 23.79, "tpot_ms": 9.230396145475833, "tpot_relative": 1.3402, "ttft_error_pct": 70.03, "ttft_ms": 83.92777933749929, "ttft_relative": 0.339419, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 7182.049514062237, "interactivity_tok_s": 134.10747785323673, "output_per_gpu": 62.03210808360334, "total_per_gpu": 99.2065076543353, "tpot_ms": 7.456705740856379, "tpot_relative": 1.08267, "ttft_ms": 280.05722372327, "ttft_relative": 1.132601, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11769.62, "interactivity_tok_s": 87.93527963418924, "output_per_gpu": 64.8151, "total_per_gpu": null, "tpot_error_pct": 36.9, "tpot_ms": 11.372, "tpot_relative": 1.651148, "ttft_error_pct": 57.23, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9738.402671437489, "interactivity_tok_s": 94.90687082968176, "output_per_gpu": 73.45910411852216, "total_per_gpu": 147.70020534659506, "tpot_error_pct": 26.84, "tpot_ms": 10.536644936851651, "tpot_relative": 1.529859, "ttft_error_pct": 69.36, "ttft_ms": 97.47933900625017, "ttft_relative": 0.394224, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 7914.118170121219, "interactivity_tok_s": 120.38219743674765, "output_per_gpu": 112.95242368778678, "total_per_gpu": 181.7983295475289, "tpot_ms": 8.306876110360333, "tpot_relative": 1.20611, "ttft_ms": 318.10005460865796, "ttft_relative": 1.286453, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14896.931, "interactivity_tok_s": 69.30487213251091, "output_per_gpu": 102.12145000000001, "total_per_gpu": null, "tpot_error_pct": 44.8, "tpot_ms": 14.429, "tpot_relative": 2.095007, "ttft_error_pct": 64.97, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12310.520856568743, "interactivity_tok_s": 75.68524107942461, "output_per_gpu": 116.68684195471795, "total_per_gpu": 233.00457291370995, "tpot_error_pct": 32.59, "tpot_ms": 13.212615639958035, "tpot_relative": 1.918395, "ttft_error_pct": 69.29, "ttft_ms": 119.26609404999954, "ttft_relative": 0.482333, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 9569.083442367264, "interactivity_tok_s": 100.35215863125603, "output_per_gpu": 186.30175975771732, "total_per_gpu": 298.0358527604411, "tpot_ms": 9.96490771737656, "tpot_relative": 1.446846, "ttft_ms": 388.3919290470658, "ttft_relative": 1.570726, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19039.058, "interactivity_tok_s": 54.11841108345059, "output_per_gpu": 159.39995, "total_per_gpu": null, "tpot_error_pct": 51.64, "tpot_ms": 18.478, "tpot_relative": 2.682898, "ttft_error_pct": 67.71, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15692.048240185992, "interactivity_tok_s": 59.23797532107557, "output_per_gpu": 182.36516368119175, "total_per_gpu": 364.81255166795484, "tpot_error_pct": 38.54, "tpot_ms": 16.88106311162566, "tpot_relative": 2.451032, "ttft_error_pct": 64.75, "ttft_ms": 148.53424337812518, "ttft_relative": 0.600699, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 11634.57652006473, "interactivity_tok_s": 82.06664446621521, "output_per_gpu": 306.14346667412, "total_per_gpu": 490.7375430127796, "tpot_ms": 12.18521856844866, "tpot_relative": 1.769223, "ttft_ms": 421.3380920875352, "ttft_relative": 1.703966, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23472.74, "interactivity_tok_s": 43.836577240049095, "output_per_gpu": 258.31835, "total_per_gpu": null, "tpot_error_pct": 49.81, "tpot_ms": 22.812, "tpot_relative": 3.312169, "ttft_error_pct": 71.44, "ttft_ms": 136.064, "ttft_relative": 0.550267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19361.218321987522, "interactivity_tok_s": 47.939408488895324, "output_per_gpu": 295.16520040048096, "total_per_gpu": 590.9788312739626, "tpot_error_pct": 36.99, "tpot_ms": 20.859664971286406, "tpot_relative": 3.028702, "ttft_error_pct": 59.28, "ttft_ms": 194.0158322617213, "ttft_relative": 0.784635, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 14456.391385557072, "interactivity_tok_s": 65.6723510828066, "output_per_gpu": 492.79893884985376, "total_per_gpu": 791.0631187685489, "tpot_ms": 15.22710826568543, "tpot_relative": 2.210887, "ttft_ms": 476.4833514796919, "ttft_relative": 1.926983, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 57.05, "tpot_shape_error_pct": 0.79, "ttft_mape_pct": 77.95, "ttft_shape_error_pct": 18.96}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.98, "tpot_shape_error_pct": 0.98, "ttft_mape_pct": 81.82, "ttft_shape_error_pct": 44.74}, "framework": "dynamo-vllm", "id": "85b4294f701768d8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9789.587, "interactivity_tok_s": 105.977108944468, "output_per_gpu": 10.886249999999999, "total_per_gpu": null, "tpot_error_pct": 57.49, "tpot_ms": 9.436, "tpot_relative": 1.57494, "ttft_error_pct": 75.2, "ttft_ms": 136.559, "ttft_relative": 0.247994, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8096.898628, "interactivity_tok_s": 114.27729926009769, "output_per_gpu": 12.170916762956294, "total_per_gpu": 24.46086044350504, "tpot_error_pct": 46.05, "tpot_ms": 8.750644322841211, "tpot_relative": 1.460548, "ttft_error_pct": 86.39, "ttft_ms": 74.95510289999916, "ttft_relative": 0.13612, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 6046.02191625163, "interactivity_tok_s": 166.9075376777073, "output_per_gpu": 18.185960279254523, "total_per_gpu": 32.576759161777744, "tpot_ms": 5.9913411575872955, "tpot_relative": 1.0, "ttft_ms": 550.6550536083523, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10073.981, "interactivity_tok_s": 102.94420424130121, "output_per_gpu": 21.03977777777778, "total_per_gpu": null, "tpot_error_pct": 56.07, "tpot_ms": 9.714, "tpot_relative": 1.62134, "ttft_error_pct": 73.65, "ttft_ms": 136.559, "ttft_relative": 0.247994, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8427.92026195008, "interactivity_tok_s": 111.23714685762408, "output_per_gpu": 23.905320050190667, "total_per_gpu": 47.63406634816065, "tpot_error_pct": 44.44, "tpot_ms": 8.989802671583544, "tpot_relative": 1.500466, "ttft_error_pct": 83.72, "ttft_ms": 84.34614374999816, "ttft_relative": 0.153174, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 6278.421027693548, "interactivity_tok_s": 160.6693984717102, "output_per_gpu": 34.77560180097787, "total_per_gpu": 61.79534568351029, "tpot_ms": 6.223960564438626, "tpot_relative": 1.038826, "ttft_ms": 518.2311289972859, "ttft_relative": 0.941118, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12735.827, "interactivity_tok_s": 81.19519324455992, "output_per_gpu": 66.46897222222222, "total_per_gpu": null, "tpot_error_pct": 56.16, "tpot_ms": 12.316, "tpot_relative": 2.055633, "ttft_error_pct": 79.66, "ttft_ms": 136.559, "ttft_relative": 0.247994, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10598.634473293776, "interactivity_tok_s": 88.05351087347694, "output_per_gpu": 75.15209768434886, "total_per_gpu": 150.06646963079552, "tpot_error_pct": 44.0, "tpot_ms": 11.356730584393032, "tpot_relative": 1.895524, "ttft_error_pct": 82.16, "ttft_ms": 119.80473678750079, "ttft_relative": 0.217568, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 7956.651502384921, "interactivity_tok_s": 126.79834714844267, "output_per_gpu": 113.9106864565705, "total_per_gpu": 202.21190991483877, "tpot_ms": 7.886538133098069, "tpot_relative": 1.316323, "ttft_ms": 671.4005949805141, "ttft_relative": 1.219276, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18795.056, "interactivity_tok_s": 54.82756730083886, "output_per_gpu": 179.51858333333334, "total_per_gpu": null, "tpot_error_pct": 58.46, "tpot_ms": 18.239, "tpot_relative": 3.044227, "ttft_error_pct": 83.29, "ttft_ms": 136.559, "ttft_relative": 0.247994, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15584.373657830458, "interactivity_tok_s": 59.74388566685067, "output_per_gpu": 203.30733201587987, "total_per_gpu": 407.061297541688, "tpot_error_pct": 45.42, "tpot_ms": 16.73811451729624, "tpot_relative": 2.793717, "ttft_error_pct": 75.02, "ttft_ms": 204.24022872656286, "ttft_relative": 0.370904, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 11395.492656626448, "interactivity_tok_s": 86.87851845191369, "output_per_gpu": 313.0638247617618, "total_per_gpu": 558.0870639313509, "tpot_ms": 11.51032519682629, "tpot_relative": 1.92116, "ttft_ms": 817.4708687425664, "ttft_relative": 1.484543, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 50.0, "points": 1, "status_counts": {"failed": 1, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 123.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 98.68, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.24, "tpot_shape_error_pct": 38.43, "ttft_mape_pct": 60.13, "ttft_shape_error_pct": 160.44}, "framework": "dynamo-vllm", "id": "1fa58d59e20b53fa", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 100764.977, "interactivity_tok_s": 10.166114307789277, "output_per_gpu": 3195.5353333333333, "total_per_gpu": null, "tpot_error_pct": 123.5, "tpot_ms": 98.366, "tpot_relative": 2.235025, "ttft_error_pct": 98.68, "ttft_ms": 136.559, "ttft_relative": 0.013227, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56733.768487245274, "interactivity_tok_s": 18.785754300728943, "output_per_gpu": 5377.851563381055, "total_per_gpu": 10759.611170872287, "tpot_error_pct": 20.95, "tpot_ms": 53.23182577561962, "tpot_relative": 1.209508, "ttft_error_pct": 25.05, "ttft_ms": 7738.57640504026, "ttft_relative": 0.749537, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 50828.53965526086, "interactivity_tok_s": 22.721519562476658, "output_per_gpu": 8741.168934220008, "total_per_gpu": 11675.26390120986, "tpot_ms": 44.01114094725624, "tpot_relative": 1.0, "ttft_ms": 10324.473169528766, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 179658.5963501179, "interactivity_tok_s": 20.795224556014116, "output_per_gpu": 4985.5371314316235, "total_per_gpu": 9972.030189775314, "tpot_error_pct": 25.53, "tpot_ms": 48.08796352770297, "tpot_relative": 1.092632, "ttft_error_pct": 95.21, "ttft_ms": 135391.94185295244, "ttft_relative": 13.11369, "unavailable_metrics": {}}, "concurrency": 12288, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 128791.5020661636, "interactivity_tok_s": 15.487080837087808, "output_per_gpu": 9918.333090943312, "total_per_gpu": 13243.325389786592, "tpot_ms": 64.56994772089277, "tpot_relative": 1.467127, "ttft_ms": 69356.0848020371, "ttft_relative": 6.717639, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 197.13, "tpot_shape_error_pct": 44.07, "ttft_mape_pct": 99.39, "ttft_shape_error_pct": 53.14}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "6747aebd83250e98", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 61325.258, "interactivity_tok_s": 16.71877351077525, "output_per_gpu": 2429.5219, "total_per_gpu": null, "tpot_error_pct": 143.48, "tpot_ms": 59.813, "tpot_relative": 2.434788, "ttft_error_pct": 99.16, "ttft_ms": 136.559, "ttft_relative": 0.008358, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (184149606400 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 38949.1870950548, "interactivity_tok_s": 40.706667480407255, "output_per_gpu": 5451.033142614536, "total_per_gpu": 8736.897907343095, "tpot_ms": 24.566000164010365, "tpot_relative": 1.0, "ttft_ms": 16338.28444264534, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 87910.982, "interactivity_tok_s": 11.654875817298166, "output_per_gpu": 2429.5219, "total_per_gpu": null, "tpot_error_pct": 250.77, "tpot_ms": 85.801, "tpot_relative": 3.492673, "ttft_error_pct": 99.61, "ttft_ms": 136.559, "ttft_relative": 0.008358, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "ValueError: no KV budget: non-KV memory (184149606400 bytes) meets/exceeds the KV-cache memory limit (capacity=198674743296 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 57382.5159842092, "interactivity_tok_s": 40.88236147218061, "output_per_gpu": 5504.527231590889, "total_per_gpu": 8820.308341803486, "tpot_ms": 24.46042655046906, "tpot_relative": 0.995702, "ttft_ms": 34863.18699817516, "ttft_relative": 2.133834, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}]}, {"aic": {"coverage_pct": 4.55, "points": 1, "status_counts": {"failed": 21, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.48, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.52, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 40.91, "points": 9, "status_counts": {"failed": 13, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.63, "tpot_shape_error_pct": 2.04, "ttft_mape_pct": 51.67, "ttft_shape_error_pct": 50.8}, "gpu": "gb300", "precisions": ["fp4"], "rows": 22, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "68e7d4a0566083f9", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 9489.93909531273, "interactivity_tok_s": 191.24722987288138, "output_per_gpu": 23.276434732838013, "total_per_gpu": 41.36162298002113, "tpot_ms": 5.2288339060632785, "tpot_relative": 1.0, "ttft_ms": 4651.690278641763, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 7779.836884967275, "interactivity_tok_s": 174.58865782499046, "output_per_gpu": 42.70217770386327, "total_per_gpu": 76.38523773277234, "tpot_ms": 5.727748941185004, "tpot_relative": 1.095416, "ttft_ms": 2534.3042540330016, "ttft_relative": 0.544814, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 8588.864487544439, "interactivity_tok_s": 158.42921371293068, "output_per_gpu": 77.37672333210487, "total_per_gpu": 137.94055491137613, "tpot_ms": 6.311967196984088, "tpot_relative": 1.207146, "ttft_ms": 2798.586990784194, "ttft_relative": 0.601628, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "0fe29bd17c93295c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 333, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 13678.862971023056, "interactivity_tok_s": 87.59056550487325, "output_per_gpu": 666.5549105203194, "total_per_gpu": 1185.5996238467826, "tpot_ms": 11.416754695394257, "tpot_relative": 1.0, "ttft_ms": 3157.918252314192, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 14584.715410713428, "interactivity_tok_s": 82.71416040574681, "output_per_gpu": 1246.7993004006607, "total_per_gpu": 2219.6785621102363, "tpot_ms": 12.089828332834314, "tpot_relative": 1.058955, "ttft_ms": 3453.397660131577, "ttft_relative": 1.093568, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 14858.669718021987, "interactivity_tok_s": 76.17568378214993, "output_per_gpu": 2268.035025284314, "total_per_gpu": 4036.886935775616, "tpot_ms": 13.127548718300153, "tpot_relative": 1.149849, "ttft_ms": 2767.9955028398363, "ttft_relative": 0.876525, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "1677a922b4bf817e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 30, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 9868.754026909202, "interactivity_tok_s": 138.3940522887358, "output_per_gpu": 133.61111696289626, "total_per_gpu": 222.79784223268723, "tpot_ms": 7.225744050861875, "tpot_relative": 1.0, "ttft_ms": 3210.0694242243962, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 11972.332686618513, "interactivity_tok_s": 114.05478907079583, "output_per_gpu": 223.33697904557107, "total_per_gpu": 372.58801528412965, "tpot_ms": 8.767716008656876, "tpot_relative": 1.2134, "ttft_ms": 3891.262145124298, "ttft_relative": 1.212205, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "ee9f163271a0c15e", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 24734.11306980783, "interactivity_tok_s": 40.632982509746554, "output_per_gpu": 9601.850019013464, "total_per_gpu": 12824.677764371738, "tpot_ms": 24.610548825947784, "tpot_relative": 1.0, "ttft_ms": 2083.0807888023537, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 37856.35652745741, "interactivity_tok_s": 26.272831735883216, "output_per_gpu": 12064.159927986626, "total_per_gpu": 16108.811100459949, "tpot_ms": 38.0621323979405, "tpot_relative": 1.546578, "ttft_ms": 2819.038593401851, "ttft_relative": 1.353303, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "58c3a4855e4e6d2e", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 15639.43510685309, "interactivity_tok_s": 66.84574260737463, "output_per_gpu": 3957.4436316977585, "total_per_gpu": 6338.7522303610285, "tpot_ms": 14.959815853548118, "tpot_relative": 1.0, "ttft_ms": 1855.7188238600631, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 19368.319064867934, "interactivity_tok_s": 52.62757058863178, "output_per_gpu": 6123.971533306729, "total_per_gpu": 9815.353674217917, "tpot_ms": 19.001447127715462, "tpot_relative": 1.270166, "ttft_ms": 1879.910803437652, "ttft_relative": 1.013036, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.25, "tpot_shape_error_pct": null, "ttft_mape_pct": 59.49, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e2f982c168f86e25", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42410.120681411936, "interactivity_tok_s": 22.92972354136378, "output_per_gpu": 5358.808513278793, "total_per_gpu": 10718.740163346401, "tpot_error_pct": 21.25, "tpot_ms": 43.6115157775916, "tpot_relative": 1.21253, "ttft_error_pct": 59.49, "ttft_ms": 2263.293126384572, "ttft_relative": 0.405095, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 38696.9692705235, "interactivity_tok_s": 27.80297504600067, "output_per_gpu": 7725.76201542735, "total_per_gpu": 11605.40368028737, "tpot_ms": 35.96737393554023, "tpot_relative": 1.0, "ttft_ms": 5587.061475627171, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.48, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.52, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.07, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.88, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f3944e766e8cf09b", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 73752.386, "interactivity_tok_s": 13.895836807292536, "output_per_gpu": 4150.886416666667, "total_per_gpu": null, "tpot_error_pct": 11.48, "tpot_ms": 71.964, "tpot_relative": 1.114762, "ttft_error_pct": 97.52, "ttft_ms": 133.214, "ttft_relative": 0.024826, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 62773.46994845628, "interactivity_tok_s": 15.50081626637001, "output_per_gpu": 7261.372718265837, "total_per_gpu": 14524.731169768276, "tpot_error_pct": 0.07, "tpot_ms": 64.51273164043383, "tpot_relative": 0.999338, "ttft_error_pct": 36.88, "ttft_ms": 3387.2386611912257, "ttft_relative": 0.631243, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 64790.725956917624, "interactivity_tok_s": 15.490547394440187, "output_per_gpu": 10519.749872110227, "total_per_gpu": 14047.139994473706, "tpot_ms": 64.55549791345118, "tpot_relative": 1.0, "ttft_ms": 5365.9824531027225, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 37.88, "tpot_shape_error_pct": 0.89, "ttft_mape_pct": 21.94, "ttft_shape_error_pct": 67.84}, "framework": "dynamo-vllm", "id": "848664ee69b10c98", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 30180.001190596056, "interactivity_tok_s": 31.771669775742048, "output_per_gpu": 3011.464648807222, "total_per_gpu": 6021.608900316983, "tpot_error_pct": 38.5, "tpot_ms": 31.47458119319586, "tpot_relative": 1.384955, "ttft_error_pct": 35.33, "ttft_ms": 1182.1048053990428, "ttft_relative": 0.646701, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 22769.449191104508, "interactivity_tok_s": 44.00232599702288, "output_per_gpu": 4942.019919389104, "total_per_gpu": 7914.628975170587, "tpot_ms": 22.726071346038804, "tpot_relative": 1.0, "ttft_ms": 1827.9000383230084, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 46094.71321824679, "interactivity_tok_s": 27.142811672388326, "output_per_gpu": 3926.1232775459757, "total_per_gpu": 7855.099639222261, "tpot_error_pct": 37.26, "tpot_ms": 36.84216698218018, "tpot_relative": 1.621141, "ttft_error_pct": 8.54, "ttft_ms": 12185.010609070381, "ttft_relative": 6.666125, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 35928.48417539315, "interactivity_tok_s": 37.25749065517784, "output_per_gpu": 6294.247302939783, "total_per_gpu": 10088.398777002854, "tpot_ms": 26.840240242025683, "tpot_relative": 1.181033, "ttft_ms": 11226.148289084453, "ttft_relative": 6.141555, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.31, "tpot_shape_error_pct": 2.33, "ttft_mape_pct": 64.97, "ttft_shape_error_pct": 46.54}, "framework": "dynamo-vllm", "id": "7757b57a7cc14eec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8845.403660737491, "interactivity_tok_s": 105.8608373245392, "output_per_gpu": 25.132891301830377, "total_per_gpu": 50.08014154501768, "tpot_error_pct": 41.29, "tpot_ms": 9.446363974377839, "tpot_relative": 1.412942, "ttft_error_pct": 74.47, "ttft_ms": 75.74610917499037, "ttft_relative": 0.255289, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 6480.763939513417, "interactivity_tok_s": 149.57519942813357, "output_per_gpu": 38.475655046903604, "total_per_gpu": 67.3019596067561, "tpot_ms": 6.685600312239398, "tpot_relative": 1.0, "ttft_ms": 296.7070467366284, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9374.618344731245, "interactivity_tok_s": 98.59034578670726, "output_per_gpu": 47.1749683767784, "total_per_gpu": 94.85213031222406, "tpot_error_pct": 42.93, "tpot_ms": 10.142980958434046, "tpot_relative": 1.517138, "ttft_error_pct": 69.68, "ttft_ms": 94.7815606187504, "ttft_relative": 0.319445, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 6805.132918188065, "interactivity_tok_s": 140.91212949644253, "output_per_gpu": 73.42180343539899, "total_per_gpu": 129.25205236288903, "tpot_ms": 7.096621160815301, "tpot_relative": 1.061479, "ttft_ms": 312.617089750529, "ttft_relative": 1.053622, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10687.198129618675, "interactivity_tok_s": 87.27823749575411, "output_per_gpu": 83.39815430529573, "total_per_gpu": 166.53249843918178, "tpot_error_pct": 39.06, "tpot_ms": 11.457609923076731, "tpot_relative": 1.713774, "ttft_error_pct": 67.23, "ttft_ms": 115.60765580311799, "ttft_relative": 0.389636, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 7964.997585234505, "interactivity_tok_s": 121.3649451705152, "output_per_gpu": 128.29321174466432, "total_per_gpu": 224.18498149849776, "tpot_ms": 8.239611517106699, "tpot_relative": 1.232442, "ttft_ms": 352.7973906559055, "ttft_relative": 1.189043, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12930.566539095298, "interactivity_tok_s": 72.10741227875756, "output_per_gpu": 137.32629250349754, "total_per_gpu": 274.7145023096743, "tpot_error_pct": 36.86, "tpot_ms": 13.868199792472575, "tpot_relative": 2.074339, "ttft_error_pct": 58.81, "ttft_ms": 161.37124621249987, "ttft_relative": 0.543874, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 9723.346292531369, "interactivity_tok_s": 98.68615351639063, "output_per_gpu": 208.87470076261943, "total_per_gpu": 366.07835647114837, "tpot_ms": 10.133133822405101, "tpot_relative": 1.515666, "ttft_ms": 391.7534688151136, "ttft_relative": 1.320338, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16263.240824917975, "interactivity_tok_s": 57.33103426639455, "output_per_gpu": 218.98796014318046, "total_per_gpu": 438.4570016143238, "tpot_error_pct": 36.43, "tpot_ms": 17.44255991185153, "tpot_relative": 2.608974, "ttft_error_pct": 54.64, "ttft_ms": 235.68815692656298, "ttft_relative": 0.794346, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 12268.11704842587, "interactivity_tok_s": 78.2185539288103, "output_per_gpu": 330.92835041876367, "total_per_gpu": 580.7157689128994, "tpot_ms": 12.784690457332392, "tpot_relative": 1.912273, "ttft_ms": 519.6355336353463, "ttft_relative": 1.751342, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "02464f0d30c30eda", "is_multinode": true, "parallelism": {"attention_dp_size": 24, "moe_ep_size": 24, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external decode prediction failed: AIC static_gen evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"nvfp4\", distribution: \"power_law_1.01\", topk: 8, num_experts: 384, hidden_size: 7168, inter_size: 2048, moe_tp_size: 1, moe_ep_size: 24 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 18147.30118749786, "interactivity_tok_s": 54.83794156839123, "output_per_gpu": 2055.740168484906, "total_per_gpu": 3529.253166785658, "tpot_ms": 18.235549537410122, "tpot_relative": 1.0, "ttft_ms": 1357.1593632859701, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 58}, {"aic": {"coverage_pct": 51.54, "points": 67, "status_counts": {"failed": 63, "success": 67, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.57, "tpot_shape_error_pct": 11.72, "ttft_mape_pct": 42.26, "ttft_shape_error_pct": 36.93}, "aisimulate": {"coverage_pct": 70.0, "points": 91, "status_counts": {"failed": 39, "success": 91, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.53, "tpot_shape_error_pct": 10.35, "ttft_mape_pct": 57.55, "ttft_shape_error_pct": 70.32}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 64.15, "points": 34, "status_counts": {"failed": 19, "success": 34, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.26, "tpot_shape_error_pct": 13.68, "ttft_mape_pct": 36.33, "ttft_shape_error_pct": 33.9}, "aisimulate": {"coverage_pct": 79.25, "points": 42, "status_counts": {"failed": 11, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.26, "tpot_shape_error_pct": 12.64, "ttft_mape_pct": 93.15, "ttft_shape_error_pct": 111.61}, "gpu": "b200", "precisions": ["fp4"], "rows": 53, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "b3c9b27232954e15", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 308, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "29528755773", "measured": {"e2e_ms": 23220.39018022792, "interactivity_tok_s": 48.83627324535026, "output_per_gpu": 1494.1259951226489, "total_per_gpu": 8970.11638132723, "tpot_ms": 20.476582948417562, "tpot_relative": 1.0, "ttft_ms": 4366.04645870049, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "446795ed577011f9", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 44005.34525323752, "interactivity_tok_s": 22.589784359471064, "output_per_gpu": 5530.686303736757, "total_per_gpu": 16601.826888095915, "tpot_ms": 44.267797517984576, "tpot_relative": 1.0, "ttft_ms": 3261.4489999496254, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "420440019e86123d", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 615, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 27874.815680981632, "interactivity_tok_s": 40.355836987768974, "output_per_gpu": 2487.772973813901, "total_per_gpu": 11192.687456789628, "tpot_ms": 24.779562874710777, "tpot_relative": 1.0, "ttft_ms": 5052.5821139407335, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "f2a002e306cf7b6c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 5711.121141259946, "interactivity_tok_s": 171.50374052510665, "output_per_gpu": 39.637211284691965, "total_per_gpu": 296.8482344181883, "tpot_ms": 5.830776617105962, "tpot_relative": 1.0, "ttft_ms": 370.47417708993635, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 30, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 10527.182841998323, "interactivity_tok_s": 132.1077151269141, "output_per_gpu": 128.2784541827641, "total_per_gpu": 962.4157824451207, "tpot_ms": 7.569580618658899, "tpot_relative": 1.298211, "ttft_ms": 3548.1994122916467, "ttft_relative": 9.577454, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 12930.386317325008, "interactivity_tok_s": 107.33956398471857, "output_per_gpu": 207.63110622082277, "total_per_gpu": 1563.373093571118, "tpot_ms": 9.31622938343932, "tpot_relative": 1.597768, "ttft_ms": 4373.1137654041695, "ttft_relative": 11.804099, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 115, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 15641.333656808261, "interactivity_tok_s": 85.6272277567475, "output_per_gpu": 331.41118681018116, "total_per_gpu": 2485.596525776545, "tpot_ms": 11.678528269545653, "tpot_relative": 2.002911, "ttft_ms": 4878.6938026687, "ttft_relative": 13.168782, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 195, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 17671.907594689488, "interactivity_tok_s": 64.91455227660154, "output_per_gpu": 498.45511747107486, "total_per_gpu": 3736.825362014382, "tpot_ms": 15.404866319326832, "tpot_relative": 2.641992, "ttft_ms": 3484.8457142215407, "ttft_relative": 9.406447, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "b12dc50c6f291de1", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 5468.217146774987, "interactivity_tok_s": 184.37242209112878, "output_per_gpu": 20.962851171993005, "total_per_gpu": 165.85552024276708, "tpot_ms": 5.423804648537596, "tpot_relative": 1.0, "ttft_ms": 451.72669868752564, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 9463.919060454129, "interactivity_tok_s": 149.40526038581908, "output_per_gpu": 71.85774643046896, "total_per_gpu": 572.904171869351, "tpot_ms": 6.693204760111082, "tpot_relative": 1.234042, "ttft_ms": 3272.068697270815, "ttft_relative": 7.24347, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "8864e7f6c7e68aaf", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=b200_sxm: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1127, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29528755773", "measured": {"e2e_ms": 31856.907381081088, "interactivity_tok_s": 32.547605878475224, "output_per_gpu": 3993.485191570452, "total_per_gpu": 14368.170279584248, "tpot_ms": 30.72422603781534, "tpot_relative": 1.0, "ttft_ms": 3546.424417469676, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.24, "tpot_shape_error_pct": 6.12, "ttft_mape_pct": 70.99, "ttft_shape_error_pct": 40.16}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.72, "tpot_shape_error_pct": 4.96, "ttft_mape_pct": 12.91, "ttft_shape_error_pct": 7.36}, "framework": "vllm", "id": "af35f5146641b49a", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 62136.43800625292, "interactivity_tok_s": 16.73851379333318, "output_per_gpu": 1053.681255327372, "total_per_gpu": null, "tpot_error_pct": 33.28, "tpot_ms": 59.74246055216039, "tpot_relative": 0.667159, "ttft_error_pct": 63.71, "ttft_ms": 1019.9008613928445, "ttft_relative": 0.362925, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 90287.77337673413, "interactivity_tok_s": 10.499674221808183, "output_per_gpu": 647.361449837841, "total_per_gpu": 5826.30007875566, "tpot_error_pct": 6.36, "tpot_ms": 95.24105023401256, "tpot_relative": 1.06358, "ttft_error_pct": 16.01, "ttft_ms": 2360.3920024394624, "ttft_relative": 0.839931, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 85553.63660605562, "interactivity_tok_s": 11.167245897780814, "output_per_gpu": 682.2287219271142, "total_per_gpu": 6140.108060634429, "tpot_ms": 89.54759384305514, "tpot_relative": 1.0, "ttft_ms": 2810.2213742881304, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 104834.48268591987, "interactivity_tok_s": 9.861807429881273, "output_per_gpu": 1249.0546683222854, "total_per_gpu": null, "tpot_error_pct": 29.2, "tpot_ms": 101.40129049468158, "tpot_relative": 1.132373, "ttft_error_pct": 78.28, "ttft_ms": 1100.9625098606264, "ttft_relative": 0.391771, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 138256.3949311558, "interactivity_tok_s": 6.907116585442071, "output_per_gpu": 847.046834460009, "total_per_gpu": 7621.858061979117, "tpot_error_pct": 1.08, "tpot_ms": 144.7782135468324, "tpot_relative": 1.616774, "ttft_error_pct": 9.82, "ttft_ms": 4571.254272827175, "ttft_relative": 1.626653, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 137380.37189994298, "interactivity_tok_s": 6.981769350588485, "output_per_gpu": 851.6075629279268, "total_per_gpu": 7662.89620016407, "tpot_ms": 143.23016842653377, "tpot_relative": 1.599487, "ttft_ms": 5069.1750795527805, "ttft_relative": 1.803835, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 90.0, "points": 9, "status_counts": {"failed": 1, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.05, "tpot_shape_error_pct": 22.64, "ttft_mape_pct": 40.78, "ttft_shape_error_pct": 44.38}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.62, "tpot_shape_error_pct": 20.12, "ttft_mape_pct": 17.27, "ttft_shape_error_pct": 16.09}, "framework": "vllm", "id": "b67176f556e06914", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6914.603261924883, "interactivity_tok_s": 152.75494914732525, "output_per_gpu": 18.493468845008216, "total_per_gpu": null, "tpot_error_pct": 9.48, "tpot_ms": 6.546432738068246, "tpot_relative": 1.094802, "ttft_error_pct": 31.84, "ttft_ms": 217.60257088106727, "ttft_relative": 1.318411, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6215.215397899998, "interactivity_tok_s": 153.31838977812785, "output_per_gpu": 18.710131919046816, "total_per_gpu": 167.48413906165135, "tpot_error_pct": 9.08, "tpot_ms": 6.522374787832909, "tpot_relative": 1.090778, "ttft_error_pct": 6.75, "ttft_ms": 153.90645810000032, "ttft_relative": 0.932489, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 5721.816370513989, "interactivity_tok_s": 167.2363899186583, "output_per_gpu": 20.32192260494675, "total_per_gpu": 181.91211725782495, "tpot_ms": 5.979559834354159, "tpot_relative": 1.0, "ttft_ms": 165.04915470140986, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7484.203524219167, "interactivity_tok_s": 141.31049778790967, "output_per_gpu": 34.171972898970914, "total_per_gpu": null, "tpot_error_pct": 7.37, "tpot_ms": 7.076615082772419, "tpot_relative": 1.183468, "ttft_error_pct": 21.02, "ttft_ms": 244.82629454298285, "ttft_relative": 1.483354, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6603.217808150007, "interactivity_tok_s": 145.21422314325594, "output_per_gpu": 35.300162390450225, "total_per_gpu": 314.57625066569136, "tpot_error_pct": 4.48, "tpot_ms": 6.8863777827980766, "tpot_relative": 1.151653, "ttft_error_pct": 16.75, "ttft_ms": 168.41803650000145, "ttft_relative": 1.020411, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 6361.601582252479, "interactivity_tok_s": 151.71856449396546, "output_per_gpu": 36.65256530564036, "total_per_gpu": 326.62814532113794, "tpot_ms": 6.591151210370004, "tpot_relative": 1.10228, "ttft_ms": 202.30789305496728, "ttft_relative": 1.225743, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8181.259142568705, "interactivity_tok_s": 129.3434046239956, "output_per_gpu": 62.520938536045705, "total_per_gpu": null, "tpot_error_pct": 1.18, "tpot_ms": 7.7313567159224235, "tpot_relative": 1.292964, "ttft_error_pct": 31.88, "ttft_ms": 272.08122218006605, "ttft_relative": 1.648486, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7106.347827800004, "interactivity_tok_s": 132.81206723113854, "output_per_gpu": 63.17368488796923, "total_per_gpu": 567.8937512188306, "tpot_error_pct": 1.46, "tpot_ms": 7.529436299336092, "tpot_relative": 1.259196, "ttft_error_pct": 10.11, "ttft_ms": 185.46297630000004, "ttft_relative": 1.123683, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 7230.377001257148, "interactivity_tok_s": 130.86928107270836, "output_per_gpu": 62.10553757914521, "total_per_gpu": 558.2917439409887, "tpot_ms": 7.641212603929718, "tpot_relative": 1.277889, "ttft_ms": 206.31199987547006, "ttft_relative": 1.250003, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11232.127994290948, "interactivity_tok_s": 93.57228612279415, "output_per_gpu": 91.07802194917733, "total_per_gpu": null, "tpot_error_pct": 14.48, "tpot_ms": 10.686924958611229, "tpot_relative": 1.787243, "ttft_error_pct": 7.81, "ttft_ms": 299.4037616316598, "ttft_relative": 1.814028, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9849.511485337487, "interactivity_tok_s": 96.25836264119212, "output_per_gpu": 92.63249983612104, "total_per_gpu": 824.0089136754076, "tpot_error_pct": 11.28, "tpot_ms": 10.388707771059334, "tpot_relative": 1.73737, "ttft_error_pct": 17.58, "ttft_ms": 228.8789125749998, "ttft_relative": 1.386732, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 8922.561719464284, "interactivity_tok_s": 107.11907004357322, "output_per_gpu": 102.01694468062375, "total_per_gpu": 907.4878893637016, "tpot_ms": 9.335405914121793, "tpot_relative": 1.56122, "ttft_ms": 277.703928075789, "ttft_relative": 1.682553, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18474.85966685665, "interactivity_tok_s": 56.369886040283355, "output_per_gpu": 110.74509018709698, "total_per_gpu": null, "tpot_error_pct": 45.59, "tpot_ms": 17.739968452045026, "tpot_relative": 2.966768, "ttft_error_pct": 0.23, "ttft_ms": 326.8719404145903, "ttft_relative": 1.980452, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16229.0593822063, "interactivity_tok_s": 57.43002065375633, "output_per_gpu": 110.48095427874605, "total_per_gpu": 997.0629593056096, "tpot_error_pct": 42.9, "tpot_ms": 17.41249591444458, "tpot_relative": 2.912003, "ttft_error_pct": 7.91, "ttft_ms": 300.3213640875009, "ttft_relative": 1.819587, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 11476.853069420758, "interactivity_tok_s": 82.06656429195702, "output_per_gpu": 155.0386295415173, "total_per_gpu": 1399.1848258961822, "tpot_ms": 12.185230472698192, "tpot_relative": 2.037814, "ttft_ms": 326.13135241954296, "ttft_relative": 1.975965, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24196.965263520768, "interactivity_tok_s": 42.906910918922236, "output_per_gpu": 169.11211614495642, "total_per_gpu": null, "tpot_error_pct": 39.52, "tpot_ms": 23.306268817385156, "tpot_relative": 3.897656, "ttft_error_pct": 24.42, "ttft_ms": 354.6522633357514, "ttft_relative": 2.148768, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21893.379208853203, "interactivity_tok_s": 43.13530365465906, "output_per_gpu": 165.77208963020996, "total_per_gpu": 1482.3868084647286, "tpot_error_pct": 38.79, "tpot_ms": 23.182866823101396, "tpot_relative": 3.877019, "ttft_error_pct": 5.98, "ttft_ms": 441.19565404375106, "ttft_relative": 2.673117, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 15934.865548436028, "interactivity_tok_s": 59.86564683176673, "output_per_gpu": 226.75929407786967, "total_per_gpu": 2027.7538093876433, "tpot_ms": 16.70407074712114, "tpot_relative": 2.793528, "ttft_ms": 469.2416720190522, "ttft_relative": 2.843042, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31068.11433636028, "interactivity_tok_s": 33.33874793501502, "output_per_gpu": 263.4212012803729, "total_per_gpu": null, "tpot_error_pct": 93.56, "tpot_ms": 29.995127649941523, "tpot_relative": 5.016277, "ttft_error_pct": 96.86, "ttft_ms": 383.09875047010013, "ttft_relative": 2.321119, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28604.60670331098, "interactivity_tok_s": 33.06807941041444, "output_per_gpu": 253.29856658864023, "total_per_gpu": 2283.1770408485513, "tpot_error_pct": 93.5, "tpot_ms": 30.240643479435352, "tpot_relative": 5.057336, "ttft_error_pct": 94.1, "ttft_ms": 721.1028495484381, "ttft_relative": 4.369019, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 441266.7269038469, "interactivity_tok_s": 2.1486808075100052, "output_per_gpu": 16.52417581375057, "total_per_gpu": 148.9452520201992, "tpot_ms": 465.40183935409567, "tpot_relative": 77.832123, "ttft_ms": 12216.899066682527, "ttft_relative": 74.019761, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44965.11603581898, "interactivity_tok_s": 22.96185270035343, "output_per_gpu": 364.0155178730404, "total_per_gpu": null, "tpot_error_pct": 4.77, "tpot_ms": 43.55049276945357, "tpot_relative": 7.283227, "ttft_error_pct": 70.28, "ttft_ms": 412.96193266798093, "ttft_relative": 2.502054, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42494.772377462614, "interactivity_tok_s": 22.356573487982686, "output_per_gpu": 341.42169682917506, "total_per_gpu": 3082.383022689702, "tpot_error_pct": 7.61, "tpot_ms": 44.72957363244995, "tpot_relative": 7.480412, "ttft_error_pct": 5.45, "ttft_ms": 1313.913653402345, "ttft_relative": 7.960741, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 39652.707674701654, "interactivity_tok_s": 24.05764528148203, "output_per_gpu": 366.4127042782456, "total_per_gpu": 3308.0038833330977, "tpot_ms": 41.56682785450051, "tpot_relative": 6.951486, "ttft_ms": 1389.6585316035726, "ttft_relative": 8.419665, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 70897.39228271238, "interactivity_tok_s": 14.520614795224365, "output_per_gpu": 461.73771624012676, "total_per_gpu": null, "tpot_error_pct": 0.49, "tpot_ms": 68.8676074740917, "tpot_relative": 11.51717, "ttft_error_pct": 82.69, "ttft_ms": 445.8298367165687, "ttft_relative": 2.701194, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 69792.48787232308, "interactivity_tok_s": 13.722769159809936, "output_per_gpu": 417.9548859965136, "total_per_gpu": 3760.5124581061373, "tpot_error_pct": 6.33, "tpot_ms": 72.87158942589473, "tpot_relative": 12.186782, "ttft_error_pct": 0.64, "ttft_ms": 2559.1848558148513, "ttft_relative": 15.505592, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 65805.56276035454, "interactivity_tok_s": 14.591224184250835, "output_per_gpu": 443.5836838281241, "total_per_gpu": 3991.1053205445455, "tpot_ms": 68.53434553348572, "tpot_relative": 11.461437, "ttft_ms": 2575.6940497103187, "ttft_relative": 15.605618, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 137228.6938149397, "interactivity_tok_s": 10.849948186642505, "output_per_gpu": 423.0447239875441, "total_per_gpu": 3807.433249698764, "tpot_error_pct": 0.8, "tpot_ms": 92.16633875091784, "tpot_relative": 15.413566, "ttft_error_pct": 7.4, "ttft_ms": 52150.80785039683, "ttft_relative": 315.971372, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 134310.5465441969, "interactivity_tok_s": 10.763094142872092, "output_per_gpu": 431.0810427634476, "total_per_gpu": 3879.7607025130783, "tpot_ms": 92.91008577326758, "tpot_relative": 15.537947, "ttft_ms": 48558.051376727504, "ttft_relative": 294.203575, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 70.0, "points": 7, "status_counts": {"failed": 3, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.46, "tpot_shape_error_pct": 8.51, "ttft_mape_pct": 26.7, "ttft_shape_error_pct": 23.96}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.13, "tpot_shape_error_pct": 14.34, "ttft_mape_pct": 254.6, "ttft_shape_error_pct": 298.16}, "framework": "vllm", "id": "23996fb0a6d7a7bb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7343.627928347851, "interactivity_tok_s": 144.06644760763294, "output_per_gpu": 34.82611081271624, "total_per_gpu": null, "tpot_error_pct": 11.07, "tpot_ms": 6.941241466045685, "tpot_relative": 1.11072, "ttft_error_pct": 28.52, "ttft_ms": 242.73790858311406, "ttft_relative": 1.285235, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6604.3984176999975, "interactivity_tok_s": 144.6177066640276, "output_per_gpu": 35.215168027521116, "total_per_gpu": 315.2293166354777, "tpot_error_pct": 10.65, "tpot_ms": 6.91478258829796, "tpot_relative": 1.106487, "ttft_error_pct": 5.53, "ttft_ms": 178.413161400001, "ttft_relative": 0.944652, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 5996.29714290495, "interactivity_tok_s": 160.01754795956037, "output_per_gpu": 38.783807063428426, "total_per_gpu": 347.174063959375, "tpot_ms": 6.249314607999867, "tpot_relative": 1.0, "ttft_ms": 188.86655640089884, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8099.453064421751, "interactivity_tok_s": 130.7122972513802, "output_per_gpu": 63.15241238286227, "total_per_gpu": null, "tpot_error_pct": 8.58, "tpot_ms": 7.650389603947084, "tpot_relative": 1.224197, "ttft_error_pct": 29.44, "ttft_ms": 273.1044995838838, "ttft_relative": 1.446018, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7142.374873400001, "interactivity_tok_s": 134.49063904203655, "output_per_gpu": 65.27487087585584, "total_per_gpu": 581.6948918163879, "tpot_error_pct": 5.53, "tpot_ms": 7.435461732674486, "tpot_relative": 1.189804, "ttft_error_pct": 7.85, "ttft_ms": 194.4235355999997, "ttft_relative": 1.029423, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 6795.302778543555, "interactivity_tok_s": 141.93038686026793, "output_per_gpu": 68.62182952514465, "total_per_gpu": 611.5212051171792, "tpot_ms": 7.045707562148134, "tpot_relative": 1.127437, "ttft_ms": 210.99503834266216, "ttft_relative": 1.117165, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9936.205459260673, "interactivity_tok_s": 106.20073276652512, "output_per_gpu": 102.95680822970007, "total_per_gpu": null, "tpot_error_pct": 12.7, "tpot_ms": 9.416130886765442, "tpot_relative": 1.506746, "ttft_error_pct": 32.89, "ttft_ms": 303.50356209962644, "ttft_relative": 1.606974, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8614.114500325031, "interactivity_tok_s": 109.42798348301802, "output_per_gpu": 104.48039243274171, "total_per_gpu": 939.2164172893563, "tpot_error_pct": 9.37, "tpot_ms": 9.13843030064781, "tpot_relative": 1.462309, "ttft_error_pct": 6.1, "ttft_ms": 214.45901295000039, "ttft_relative": 1.135505, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 7908.997534376977, "interactivity_tok_s": 119.68535575251741, "output_per_gpu": 113.73227914876597, "total_per_gpu": 1022.3853611673676, "tpot_ms": 8.355241071161426, "tpot_relative": 1.336985, "ttft_ms": 228.38428105023922, "ttft_relative": 1.209236, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13129.451719252851, "interactivity_tok_s": 79.95011529205546, "output_per_gpu": 155.8328591132083, "total_per_gpu": null, "tpot_error_pct": 20.37, "tpot_ms": 12.507799348969403, "tpot_relative": 2.001467, "ttft_error_pct": 15.32, "ttft_ms": 333.97298525715007, "ttft_relative": 1.768301, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11551.845443587526, "interactivity_tok_s": 82.04548954908982, "output_per_gpu": 158.0084398892987, "total_per_gpu": 1405.5581263063002, "tpot_error_pct": 17.3, "tpot_ms": 12.188360450962701, "tpot_relative": 1.950352, "ttft_error_pct": 8.55, "ttft_ms": 264.838972975, "ttft_relative": 1.402254, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 9910.394654439006, "interactivity_tok_s": 96.23890511288035, "output_per_gpu": 183.8898246795439, "total_per_gpu": 1635.7850099934994, "tpot_ms": 10.390808154218734, "tpot_relative": 1.662712, "ttft_ms": 289.59603595067165, "ttft_relative": 1.533337, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18080.084269810683, "interactivity_tok_s": 57.74607325073119, "output_per_gpu": 226.32637873445307, "total_per_gpu": null, "tpot_error_pct": 22.84, "tpot_ms": 17.31719481007893, "tpot_relative": 2.771055, "ttft_error_pct": 1.06, "ttft_ms": 364.5939790999365, "ttft_relative": 1.930432, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15947.838049625014, "interactivity_tok_s": 58.634847495215276, "output_per_gpu": 224.1566517878449, "total_per_gpu": 2022.9576766302566, "tpot_error_pct": 20.98, "tpot_ms": 17.054704543771553, "tpot_relative": 2.729052, "ttft_error_pct": 6.42, "ttft_ms": 344.84238758749905, "ttft_relative": 1.825852, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 13269.078338216787, "interactivity_tok_s": 70.93595757325768, "output_per_gpu": 268.8129707037181, "total_per_gpu": 2425.969777499856, "tpot_ms": 14.097222821969662, "tpot_relative": 2.255803, "ttft_ms": 368.49309132885537, "ttft_relative": 1.951076, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26320.359152741352, "interactivity_tok_s": 39.4602557876602, "output_per_gpu": 310.93800629797295, "total_per_gpu": null, "tpot_error_pct": 27.02, "tpot_ms": 25.341954329467743, "tpot_relative": 4.055157, "ttft_error_pct": 27.46, "ttft_ms": 395.5398736958524, "ttft_relative": 2.094282, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23811.36619318126, "interactivity_tok_s": 39.709030534177224, "output_per_gpu": 304.9826706403324, "total_per_gpu": 2727.2521494787366, "tpot_error_pct": 26.22, "tpot_ms": 25.183188472438495, "tpot_relative": 4.029752, "ttft_error_pct": 7.03, "ttft_ms": 506.9596668656239, "ttft_relative": 2.684221, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 19013.750400970275, "interactivity_tok_s": 50.12147246132798, "output_per_gpu": 380.8880860631314, "total_per_gpu": 3406.022543003937, "tpot_ms": 19.951528773851685, "tpot_relative": 3.192595, "ttft_ms": 545.2706731877697, "ttft_relative": 2.887068, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 39380.48074078267, "interactivity_tok_s": 26.26221561149143, "output_per_gpu": 415.63738410763483, "total_per_gpu": null, "tpot_error_pct": 26.66, "tpot_ms": 38.07751846963113, "tpot_relative": 6.093071, "ttft_error_pct": 52.2, "ttft_ms": 427.1793463500231, "ttft_relative": 2.261805, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46637.65531619076, "interactivity_tok_s": 35.67004470156808, "output_per_gpu": 306.0442021355021, "total_per_gpu": 2758.614488866708, "tpot_error_pct": 6.74, "tpot_ms": 28.034727973190325, "tpot_relative": 4.486048, "ttft_error_pct": 2226.39, "ttft_ms": 20792.533720709424, "ttft_relative": 110.091136, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 28616.07900552408, "interactivity_tok_s": 33.264197712587006, "output_per_gpu": 507.3441117471724, "total_per_gpu": 4573.087180678878, "tpot_ms": 30.062351379711917, "tpot_relative": 4.810504, "ttft_ms": 893.768601620468, "ttft_relative": 4.732276, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 91953.3639136863, "interactivity_tok_s": 35.61182641928542, "output_per_gpu": 307.93020850930145, "total_per_gpu": 2780.0191250214175, "tpot_error_pct": 17.6, "tpot_ms": 28.080559200368747, "tpot_relative": 4.493382, "ttft_error_pct": 132.29, "ttft_ms": 66122.64696431368, "ttft_relative": 350.102465, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 59840.64721008117, "interactivity_tok_s": 29.342838536887246, "output_per_gpu": 476.5277556228645, "total_per_gpu": 4302.131579257121, "tpot_ms": 34.07986581607937, "tpot_relative": 5.453377, "ttft_ms": 28465.659628930145, "ttft_relative": 150.718371, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 182976.35812503545, "interactivity_tok_s": 35.51293954703383, "output_per_gpu": 308.27709703743824, "total_per_gpu": 2773.696163867196, "tpot_error_pct": 18.36, "tpot_ms": 28.1587503809868, "tpot_relative": 4.505894, "ttft_error_pct": 80.42, "ttft_ms": 157024.69112476043, "ttft_relative": 831.405486, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 118859.05534783544, "interactivity_tok_s": 28.991596274072496, "output_per_gpu": 477.3942898178457, "total_per_gpu": 4295.313284849876, "tpot_ms": 34.492754056951014, "tpot_relative": 5.519446, "ttft_ms": 87030.6073502533, "ttft_relative": 460.804756, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 364443.50124425197, "interactivity_tok_s": 35.437190435844435, "output_per_gpu": 308.61570382554106, "total_per_gpu": 2777.5637550774613, "tpot_error_pct": 18.58, "tpot_ms": 28.218941391823996, "tpot_relative": 4.515526, "ttft_error_pct": 65.41, "ttft_ms": 338426.8468596131, "ttft_relative": 1791.883398, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 236571.16139773862, "interactivity_tok_s": 28.85454717488173, "output_per_gpu": 477.41627095320797, "total_per_gpu": 4296.781122432716, "tpot_ms": 34.656582684843286, "tpot_relative": 5.545661, "ttft_ms": 204602.32620860566, "ttft_relative": 1083.316867, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 90.0, "points": 9, "status_counts": {"failed": 1, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.75, "tpot_shape_error_pct": 10.81, "ttft_mape_pct": 40.34, "ttft_shape_error_pct": 38.02}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.93, "tpot_shape_error_pct": 9.21, "ttft_mape_pct": 5.18, "ttft_shape_error_pct": 5.74}, "framework": "vllm", "id": "ff02b19fc6a3a412", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6911.214643565913, "interactivity_tok_s": 153.1568258620019, "output_per_gpu": 18.502536326092393, "total_per_gpu": null, "tpot_error_pct": 11.4, "tpot_ms": 6.529255189063691, "tpot_relative": 1.113986, "ttft_error_pct": 38.03, "ttft_ms": 231.78658515375707, "ttft_relative": 1.380301, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6213.7208128, "interactivity_tok_s": 153.7232405449276, "output_per_gpu": 18.714632263563033, "total_per_gpu": 167.52442399016178, "tpot_error_pct": 10.99, "tpot_ms": 6.50519723924072, "tpot_relative": 1.109881, "ttft_error_pct": 0.27, "ttft_ms": 168.37496890000006, "ttft_relative": 1.002682, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 5614.747073117178, "interactivity_tok_s": 170.6144990269846, "output_per_gpu": 20.7091422379265, "total_per_gpu": 185.37832193975783, "tpot_ms": 5.861166581404309, "tpot_relative": 1.0, "ttft_ms": 167.9246677143965, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7576.333349027354, "interactivity_tok_s": 139.83911317541776, "output_per_gpu": 33.756434441052285, "total_per_gpu": null, "tpot_error_pct": 14.75, "tpot_ms": 7.151075098320843, "tpot_relative": 1.220077, "ttft_error_pct": 35.01, "ttft_ms": 260.78352344513246, "ttft_relative": 1.552979, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6675.645583550002, "interactivity_tok_s": 143.93531060469445, "output_per_gpu": 34.91861219623423, "total_per_gpu": 311.17607850190257, "tpot_error_pct": 11.49, "tpot_ms": 6.947565512582324, "tpot_relative": 1.185355, "ttft_error_pct": 4.96, "ttft_ms": 183.57874914999974, "ttft_relative": 1.093221, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 6016.219607152743, "interactivity_tok_s": 160.46679436190368, "output_per_gpu": 38.74428721658981, "total_per_gpu": 345.26845719573055, "tpot_ms": 6.2318188879917535, "tpot_relative": 1.063239, "ttft_ms": 193.16220019536559, "ttft_relative": 1.150291, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8478.053798389275, "interactivity_tok_s": 124.93524481564461, "output_per_gpu": 60.3322427721771, "total_per_gpu": null, "tpot_error_pct": 10.37, "tpot_ms": 8.00414647984728, "tpot_relative": 1.365623, "ttft_error_pct": 36.93, "ttft_ms": 289.811949505507, "ttft_relative": 1.725845, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7360.215055024984, "interactivity_tok_s": 128.41038071474173, "output_per_gpu": 61.035558415982074, "total_per_gpu": 548.673269385131, "tpot_error_pct": 7.38, "tpot_ms": 7.787532397567281, "tpot_relative": 1.328666, "ttft_error_pct": 4.5, "ttft_ms": 202.1244957999996, "ttft_relative": 1.203662, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 6878.355449825176, "interactivity_tok_s": 137.88825415879566, "output_per_gpu": 65.35775112010278, "total_per_gpu": 587.5272040983997, "tpot_ms": 7.252249338427147, "tpot_relative": 1.237339, "ttft_ms": 211.64613964647288, "ttft_relative": 1.260364, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11050.771392794783, "interactivity_tok_s": 95.32361901821263, "output_per_gpu": 92.57272308311495, "total_per_gpu": null, "tpot_error_pct": 16.12, "tpot_ms": 10.490579462881481, "tpot_relative": 1.789845, "ttft_error_pct": 24.79, "ttft_ms": 318.908602267027, "ttft_relative": 1.899117, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9686.384213625006, "interactivity_tok_s": 98.1297779966731, "output_per_gpu": 94.15748934419729, "total_per_gpu": 837.5744004121295, "tpot_error_pct": 12.8, "tpot_ms": 10.190586592725229, "tpot_relative": 1.738662, "ttft_error_pct": 2.73, "ttft_ms": 248.5836828625002, "ttft_relative": 1.480329, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 8621.578313138889, "interactivity_tok_s": 110.69141743036464, "output_per_gpu": 105.71326509264618, "total_per_gpu": 940.3683683823533, "tpot_ms": 9.034124083098805, "tpot_relative": 1.541353, "ttft_ms": 255.55370803776896, "ttft_relative": 1.521835, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18216.938046906133, "interactivity_tok_s": 57.25067219659438, "output_per_gpu": 112.31305693261012, "total_per_gpu": null, "tpot_error_pct": 46.61, "tpot_ms": 17.46704382030795, "tpot_relative": 2.980131, "ttft_error_pct": 7.97, "ttft_ms": 348.15221873109715, "ttft_relative": 2.073264, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16006.134860600034, "interactivity_tok_s": 58.340058929554075, "output_per_gpu": 111.99558757524652, "total_per_gpu": 1010.7321456982374, "tpot_error_pct": 43.87, "tpot_ms": 17.140880868966985, "tpot_relative": 2.924483, "ttft_error_pct": 0.7, "ttft_ms": 324.7162821250002, "ttft_relative": 1.933702, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 11223.751098063804, "interactivity_tok_s": 83.9353721239513, "output_per_gpu": 158.73558826607132, "total_per_gpu": 1432.5489530473146, "tpot_ms": 11.913928236634886, "tpot_relative": 2.032689, "ttft_ms": 322.4611110330443, "ttft_relative": 1.920272, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24496.45687583485, "interactivity_tok_s": 42.41514038565837, "output_per_gpu": 167.04456569948516, "total_per_gpu": null, "tpot_error_pct": 39.71, "tpot_ms": 23.57648686076553, "tpot_relative": 4.02249, "ttft_error_pct": 21.69, "ttft_ms": 377.7108172717128, "ttft_relative": 2.249287, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22179.700369034366, "interactivity_tok_s": 42.63930376157724, "output_per_gpu": 163.69922795767712, "total_per_gpu": 1463.850619375303, "tpot_error_pct": 38.98, "tpot_ms": 23.45254053845765, "tpot_relative": 4.001343, "ttft_error_pct": 1.31, "ttft_ms": 475.9706939999995, "ttft_relative": 2.83443, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 16107.692767992557, "interactivity_tok_s": 59.25811665343034, "output_per_gpu": 224.79846416429058, "total_per_gpu": 2010.2194439584764, "tpot_ms": 16.87532538113683, "tpot_relative": 2.879175, "ttft_ms": 482.3121346054904, "ttft_relative": 2.872193, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32849.04057425131, "interactivity_tok_s": 31.534073549523953, "output_per_gpu": 249.13969653089413, "total_per_gpu": null, "tpot_error_pct": 25.56, "tpot_ms": 31.71172916906881, "tpot_relative": 5.410481, "ttft_error_pct": 48.35, "ttft_ms": 407.94163429392086, "ttft_relative": 2.429313, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30309.086699048425, "interactivity_tok_s": 31.223771938338935, "output_per_gpu": 239.31889643951234, "total_per_gpu": 2157.167397947729, "tpot_error_pct": 26.81, "tpot_ms": 32.026880095550645, "tpot_relative": 5.46425, "ttft_error_pct": 1.54, "ttft_ms": 777.5662008171873, "ttft_relative": 4.630447, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 24082.05089976709, "interactivity_tok_s": 39.59474120032999, "output_per_gpu": 301.4826446799421, "total_per_gpu": 2717.497622737892, "tpot_ms": 25.25587918204819, "tpot_relative": 4.309019, "ttft_ms": 789.746244834123, "ttft_relative": 4.702979, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 47529.88862489698, "interactivity_tok_s": 21.724225388624124, "output_per_gpu": 344.37278254900343, "total_per_gpu": null, "tpot_error_pct": 9.41, "tpot_ms": 46.03156071671257, "tpot_relative": 7.853652, "ttft_error_pct": 68.08, "ttft_ms": 439.60201170002136, "ttft_relative": 2.617853, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 45058.6799536856, "interactivity_tok_s": 21.096451464070423, "output_per_gpu": 322.2227649533272, "total_per_gpu": 2909.0534943747557, "tpot_error_pct": 12.67, "tpot_ms": 47.40133674628219, "tpot_relative": 8.087355, "ttft_error_pct": 2.78, "ttft_ms": 1415.665416649999, "ttft_relative": 8.43036, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 40107.40790350069, "interactivity_tok_s": 23.76913750241772, "output_per_gpu": 362.37464921817167, "total_per_gpu": 3271.547991755458, "tpot_ms": 42.07136249257186, "tpot_relative": 7.177984, "ttft_ms": 1377.3698308199982, "ttft_relative": 8.202308, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 76353.07957001301, "interactivity_tok_s": 13.482029227318387, "output_per_gpu": 428.7449855900347, "total_per_gpu": null, "tpot_error_pct": 3.83, "tpot_ms": 74.17281057169929, "tpot_relative": 12.654957, "ttft_error_pct": 82.18, "ttft_ms": 474.2943551646505, "ttft_relative": 2.824447, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 75550.6527907101, "interactivity_tok_s": 12.675398524782969, "output_per_gpu": 386.32249076166136, "total_per_gpu": 3475.9027541741407, "tpot_error_pct": 10.43, "tpot_ms": 78.89298297365544, "tpot_relative": 13.460287, "ttft_error_pct": 3.73, "ttft_ms": 2760.949809214054, "ttft_relative": 16.441598, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 68568.30902849336, "interactivity_tok_s": 13.997995817844844, "output_per_gpu": 425.87601256647866, "total_per_gpu": 3831.7821002292767, "tpot_ms": 71.43879831176874, "tpot_relative": 12.188495, "ttft_ms": 2661.6234236669243, "ttft_relative": 15.850104, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 151068.69583775842, "interactivity_tok_s": 10.41529722215619, "output_per_gpu": 383.8158091296847, "total_per_gpu": 3454.370166033375, "tpot_error_pct": 3.86, "tpot_ms": 96.01262246004138, "tpot_relative": 16.381145, "ttft_error_pct": 29.25, "ttft_ms": 62415.364330456985, "ttft_relative": 371.686692, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 133618.52997434666, "interactivity_tok_s": 10.817298220208956, "output_per_gpu": 433.2883482396439, "total_per_gpu": 3899.626612157561, "tpot_ms": 92.44452539283725, "tpot_relative": 15.772376, "ttft_ms": 48290.11475325705, "ttft_relative": 287.570108, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 70.0, "points": 7, "status_counts": {"failed": 3, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.69, "tpot_shape_error_pct": 12.0, "ttft_mape_pct": 25.2, "ttft_shape_error_pct": 23.34}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.65, "tpot_shape_error_pct": 7.73, "ttft_mape_pct": 111.6, "ttft_shape_error_pct": 138.03}, "framework": "vllm", "id": "fdddf64a3b075cba", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7212.12246421423, "interactivity_tok_s": 146.606106902046, "output_per_gpu": 35.46112829739148, "total_per_gpu": null, "tpot_error_pct": 4.95, "tpot_ms": 6.820998259425469, "tpot_relative": 1.049466, "ttft_error_pct": 24.11, "ttft_ms": 234.24124482197513, "ttft_relative": 1.241073, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6483.1315807999945, "interactivity_tok_s": 147.17701140606553, "output_per_gpu": 35.873866988721694, "total_per_gpu": 321.12567422904306, "tpot_error_pct": 4.54, "tpot_ms": 6.794539381160362, "tpot_relative": 1.045395, "ttft_error_pct": 10.52, "ttft_ms": 168.8883368999996, "ttft_relative": 0.894815, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 6228.6830488126725, "interactivity_tok_s": 153.85806809208745, "output_per_gpu": 37.336493581954535, "total_per_gpu": 334.2184069150646, "tpot_ms": 6.4994966620891015, "tpot_relative": 1.0, "ttft_ms": 188.7409788032528, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7921.049139464142, "interactivity_tok_s": 133.5944421777265, "output_per_gpu": 64.57477929932435, "total_per_gpu": null, "tpot_error_pct": 1.26, "tpot_ms": 7.485341333808307, "tpot_relative": 1.15168, "ttft_error_pct": 29.88, "ttft_ms": 263.5449549782443, "ttft_relative": 1.396331, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6984.3687896499905, "interactivity_tok_s": 137.4160037421765, "output_per_gpu": 66.75057907167957, "total_per_gpu": 594.8456174754983, "tpot_error_pct": 1.56, "tpot_ms": 7.277172765671646, "tpot_relative": 1.119652, "ttft_error_pct": 9.14, "ttft_ms": 184.36210499999925, "ttft_relative": 0.9768, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 7111.0112523514545, "interactivity_tok_s": 135.27570987422905, "output_per_gpu": 65.57248771519056, "total_per_gpu": 584.3470946141352, "tpot_ms": 7.392310126701519, "tpot_relative": 1.137367, "ttft_ms": 202.91656115441583, "ttft_relative": 1.075106, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9689.92555987005, "interactivity_tok_s": 108.86400426702816, "output_per_gpu": 105.573566450981, "total_per_gpu": null, "tpot_error_pct": 6.02, "tpot_ms": 9.185772714616853, "tpot_relative": 1.413305, "ttft_error_pct": 29.23, "ttft_ms": 292.8800728170086, "ttft_relative": 1.551757, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8395.373893474989, "interactivity_tok_s": 112.20406451772409, "output_per_gpu": 107.18909303488306, "total_per_gpu": 963.5660202705131, "tpot_error_pct": 2.87, "tpot_ms": 8.912333116435697, "tpot_relative": 1.371234, "ttft_error_pct": 10.2, "ttft_ms": 203.51812887499858, "ttft_relative": 1.078293, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 8190.864226598933, "interactivity_tok_s": 115.42075732834371, "output_per_gpu": 109.73747968420336, "total_per_gpu": 986.4744964248637, "tpot_ms": 8.66395285516318, "tpot_relative": 1.333019, "ttft_ms": 226.6274742767564, "ttft_relative": 1.200733, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13699.284011270454, "interactivity_tok_s": 76.47454148922138, "output_per_gpu": 149.3508710613451, "total_per_gpu": null, "tpot_error_pct": 20.23, "tpot_ms": 13.07624708205598, "tpot_relative": 2.011886, "ttft_error_pct": 13.34, "ttft_ms": 322.28324632718574, "ttft_relative": 1.707543, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12053.904047812466, "interactivity_tok_s": 78.4663504605401, "output_per_gpu": 151.53327224923035, "total_per_gpu": 1347.9585164242583, "tpot_error_pct": 17.18, "tpot_ms": 12.744316437947365, "tpot_relative": 1.960816, "ttft_error_pct": 11.22, "ttft_ms": 252.44253932499996, "ttft_relative": 1.337508, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 10354.664682386647, "interactivity_tok_s": 91.94800844406083, "output_per_gpu": 175.9429072689448, "total_per_gpu": 1565.093505455019, "tpot_ms": 10.87571136038665, "tpot_relative": 1.673316, "ttft_ms": 284.3409904882719, "ttft_relative": 1.506514, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18618.153446596792, "interactivity_tok_s": 56.00471120855091, "output_per_gpu": 219.78549117329226, "total_per_gpu": null, "tpot_error_pct": 22.8, "tpot_ms": 17.855640684873634, "tpot_relative": 2.747234, "ttft_error_pct": 0.46, "ttft_ms": 351.8330259710632, "ttft_relative": 1.864105, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16414.689457106255, "interactivity_tok_s": 56.874633953579725, "output_per_gpu": 217.89284240518344, "total_per_gpu": 1966.428364764925, "tpot_error_pct": 20.93, "tpot_ms": 17.582530743251656, "tpot_relative": 2.705214, "ttft_error_pct": 6.68, "ttft_ms": 329.8440700625006, "ttft_relative": 1.747602, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 13658.010702495085, "interactivity_tok_s": 68.77592725118434, "output_per_gpu": 260.9211916229335, "total_per_gpu": 2354.748446585022, "tpot_ms": 14.539971178400648, "tpot_relative": 2.237092, "ttft_ms": 353.4417839196976, "ttft_relative": 1.872629, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26715.90070569698, "interactivity_tok_s": 38.84681739672105, "output_per_gpu": 306.3344219667211, "total_per_gpu": null, "tpot_error_pct": 24.99, "tpot_ms": 25.742134543160983, "tpot_relative": 3.960635, "ttft_error_pct": 27.56, "ttft_ms": 381.69706804329337, "ttft_relative": 2.022333, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24146.155284978162, "interactivity_tok_s": 39.108932819236415, "output_per_gpu": 300.7423074479953, "total_per_gpu": 2689.3334716515956, "tpot_error_pct": 24.15, "tpot_ms": 25.569605916429722, "tpot_relative": 3.93409, "ttft_error_pct": 7.72, "ttft_ms": 486.20897337187535, "ttft_relative": 2.576065, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 19586.911604810302, "interactivity_tok_s": 48.5552911070814, "output_per_gpu": 369.1284682508098, "total_per_gpu": 3300.8642961816954, "tpot_ms": 20.595077842178934, "tpot_relative": 3.168719, "ttft_ms": 526.9025392466574, "ttft_relative": 2.79167, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 38793.58638195475, "interactivity_tok_s": 26.6535669803396, "output_per_gpu": 421.92541413530535, "total_per_gpu": null, "tpot_error_pct": 22.56, "tpot_ms": 37.5184304876577, "tpot_relative": 5.772513, "ttft_error_pct": 51.79, "ttft_ms": 412.2319930809215, "ttft_relative": 2.184115, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 40044.73788489691, "interactivity_tok_s": 29.369540218754615, "output_per_gpu": 359.50283703860924, "total_per_gpu": 3240.478754779033, "tpot_error_pct": 11.22, "tpot_ms": 34.04888168325585, "tpot_relative": 5.238695, "ttft_error_pct": 913.2, "ttft_ms": 8663.26778094689, "ttft_relative": 45.900301, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 29082.64402013301, "interactivity_tok_s": 32.665469139924944, "output_per_gpu": 498.43123143338755, "total_per_gpu": 4492.748456404482, "tpot_ms": 30.613367152831216, "tpot_relative": 4.710114, "ttft_ms": 855.0395725725139, "ttft_relative": 4.530228, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 78666.27239677089, "interactivity_tok_s": 29.191161634296613, "output_per_gpu": 361.10897800927023, "total_per_gpu": 3260.121408492424, "tpot_error_pct": 1.75, "tpot_ms": 34.256944363087726, "tpot_relative": 5.270707, "ttft_error_pct": 66.04, "ttft_ms": 47140.62921855901, "ttft_relative": 249.763615, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 59387.49432149766, "interactivity_tok_s": 29.70096392279859, "output_per_gpu": 479.7696006510942, "total_per_gpu": 4331.399221501328, "tpot_ms": 33.66894093401446, "tpot_relative": 5.180238, "ttft_ms": 28391.08986671854, "ttft_relative": 150.42356, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 156344.9120389864, "interactivity_tok_s": 28.989166952867226, "output_per_gpu": 362.55909950965236, "total_per_gpu": 3262.093723955564, "tpot_error_pct": 1.39, "tpot_ms": 34.49564458426403, "tpot_relative": 5.307433, "ttft_error_pct": 43.94, "ttft_ms": 124531.46759939282, "ttft_relative": 659.8009, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 117908.15754336813, "interactivity_tok_s": 29.39147713319216, "output_per_gpu": 481.58919030211564, "total_per_gpu": 4333.056534325237, "tpot_ms": 34.02346862215671, "tpot_relative": 5.234785, "ttft_ms": 86518.79215195624, "ttft_relative": 458.399616, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 310430.42656430055, "interactivity_tok_s": 28.992418375509647, "output_per_gpu": 363.20429884916354, "total_per_gpu": 3268.865076101383, "tpot_error_pct": 0.87, "tpot_ms": 34.49177598943301, "tpot_relative": 5.306838, "ttft_error_pct": 37.34, "ttft_ms": 278621.46671944886, "ttft_relative": 1476.210776, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31147925932", "measured": {"e2e_ms": 234411.4798320561, "interactivity_tok_s": 29.245719739035334, "output_per_gpu": 481.02896537185666, "total_per_gpu": 4329.295634659487, "tpot_ms": 34.19303778204724, "tpot_relative": 5.260875, "ttft_ms": 202865.4884137697, "ttft_relative": 1074.835416, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 92.0, "points": 23, "status_counts": {"failed": 2, "success": 23, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 41.29, "tpot_shape_error_pct": 7.58, "ttft_mape_pct": 48.64, "ttft_shape_error_pct": 40.44}, "aisimulate": {"coverage_pct": 100.0, "points": 25, "status_counts": {"failed": 0, "success": 25, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.72, "tpot_shape_error_pct": 6.89, "ttft_mape_pct": 14.14, "ttft_shape_error_pct": 4.76}, "gpu": "b300", "precisions": ["fp4"], "rows": 25, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.08, "tpot_shape_error_pct": 5.55, "ttft_mape_pct": 63.16, "ttft_shape_error_pct": 49.35}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.93, "tpot_shape_error_pct": 10.0, "ttft_mape_pct": 14.03, "ttft_shape_error_pct": 4.03}, "framework": "vllm", "id": "e2293f1e9f5bae18", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 42128.73826435317, "interactivity_tok_s": 24.670125701142204, "output_per_gpu": 777.0467701782386, "total_per_gpu": null, "tpot_error_pct": 17.01, "tpot_ms": 40.534856291944266, "tpot_relative": 0.829864, "ttft_error_pct": 45.09, "ttft_ms": 661.5802776941872, "ttft_relative": 0.549107, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52113.359219622755, "interactivity_tok_s": 18.009619540375496, "output_per_gpu": 555.7838333947719, "total_per_gpu": 5017.661936108849, "tpot_error_pct": 13.68, "tpot_ms": 55.52588147451505, "tpot_relative": 1.136773, "ttft_error_pct": 16.28, "ttft_ms": 1008.6726130890635, "ttft_relative": 0.837191, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 46169.52359226114, "interactivity_tok_s": 20.472841489286555, "output_per_gpu": 627.4696573462118, "total_per_gpu": 5664.847421880705, "tpot_ms": 48.8451981872326, "tpot_relative": 1.0, "ttft_ms": 1204.8301284700528, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 57367.15587044648, "interactivity_tok_s": 18.057587823423194, "output_per_gpu": 1141.2802152481966, "total_per_gpu": null, "tpot_error_pct": 19.77, "tpot_ms": 55.37838219470606, "tpot_relative": 1.133753, "ttft_error_pct": 64.81, "ttft_ms": 715.0708852621727, "ttft_relative": 0.593503, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 69178.4421285574, "interactivity_tok_s": 13.683781439798006, "output_per_gpu": 840.7050566606399, "total_per_gpu": 7564.170069760912, "tpot_error_pct": 5.87, "tpot_ms": 73.07921457233985, "tpot_relative": 1.496139, "ttft_error_pct": 14.36, "ttft_ms": 1740.2466000777283, "ttft_relative": 1.444392, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 65764.00073551796, "interactivity_tok_s": 14.487159757801617, "output_per_gpu": 885.5230853587971, "total_per_gpu": 7967.416355219061, "tpot_ms": 69.02664267655919, "tpot_relative": 1.413172, "ttft_ms": 2032.1580867102282, "ttft_relative": 1.686676, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 86198.49346797411, "interactivity_tok_s": 11.97516949231341, "output_per_gpu": 1519.0984752958643, "total_per_gpu": null, "tpot_error_pct": 23.47, "tpot_ms": 83.5061249564674, "tpot_relative": 1.709608, "ttft_error_pct": 79.57, "ttft_ms": 771.7276375079476, "ttft_relative": 0.640528, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 102830.48100513851, "interactivity_tok_s": 9.280935243616302, "output_per_gpu": 1133.8933749890255, "total_per_gpu": 10205.122751214074, "tpot_error_pct": 1.25, "tpot_ms": 107.74776180965482, "tpot_relative": 2.205903, "ttft_error_pct": 11.45, "ttft_ms": 3344.7388939279285, "ttft_relative": 2.776108, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 104554.4811279246, "interactivity_tok_s": 9.165050385208971, "output_per_gpu": 1116.9556885332624, "total_per_gpu": 10052.682342604834, "tpot_ms": 109.11014756818483, "tpot_relative": 2.233795, "ttft_ms": 3777.145193163824, "ttft_relative": 3.135002, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.96, "tpot_shape_error_pct": null, "ttft_mape_pct": 51.67, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.46, "tpot_shape_error_pct": null, "ttft_mape_pct": 8.06, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "78163362a7b68b9e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9237.301062297633, "interactivity_tok_s": 113.46470752862872, "output_per_gpu": 13.843329251433223, "total_per_gpu": null, "tpot_error_pct": 44.96, "tpot_ms": 8.813313159492225, "tpot_relative": 1.44964, "ttft_error_pct": 51.67, "ttft_ms": 221.2817001370861, "ttft_relative": 1.516742, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8319.351202500007, "interactivity_tok_s": 113.86199353431094, "output_per_gpu": 13.977952988095396, "total_per_gpu": 125.12393991579405, "tpot_error_pct": 44.46, "tpot_ms": 8.782561844911507, "tpot_relative": 1.444582, "ttft_error_pct": 8.06, "ttft_ms": 157.6586747000007, "ttft_relative": 1.080647, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 5795.71858300078, "interactivity_tok_s": 164.48297839736657, "output_per_gpu": 20.0633278488576, "total_per_gpu": 179.59730086439487, "tpot_ms": 6.079656446785319, "tpot_relative": 1.0, "ttft_ms": 145.89281610096805, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 90.0, "points": 9, "status_counts": {"failed": 1, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.01, "tpot_shape_error_pct": 8.35, "ttft_mape_pct": 45.41, "ttft_shape_error_pct": 39.26}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.75, "tpot_shape_error_pct": 6.81, "ttft_mape_pct": 12.83, "ttft_shape_error_pct": 4.09}, "framework": "vllm", "id": "643414cc11473775", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9794.767105111772, "interactivity_tok_s": 107.13794205177304, "output_per_gpu": 26.110881173124284, "total_per_gpu": null, "tpot_error_pct": 37.52, "tpot_ms": 9.33376151202123, "tpot_relative": 1.375231, "ttft_error_pct": 49.22, "ttft_ms": 246.32907831405393, "ttft_relative": 1.492227, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8822.226113900002, "interactivity_tok_s": 107.54131951557827, "output_per_gpu": 26.362393912525395, "total_per_gpu": 235.98352310646726, "tpot_error_pct": 37.01, "tpot_ms": 9.298751442743285, "tpot_relative": 1.370073, "ttft_error_pct": 9.52, "ttft_ms": 180.7952925999993, "ttft_relative": 1.095232, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 6472.490247007227, "interactivity_tok_s": 147.33941241155526, "output_per_gpu": 35.93148605134826, "total_per_gpu": 321.64145247899364, "tpot_ms": 6.787050278215809, "tpot_relative": 1.0, "ttft_ms": 165.0748130166903, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10887.990272905492, "interactivity_tok_s": 96.41079135126337, "output_per_gpu": 46.978366730622064, "total_per_gpu": null, "tpot_error_pct": 48.91, "tpot_ms": 10.37228287398448, "tpot_relative": 1.528246, "ttft_error_pct": 53.28, "ttft_ms": 277.1448928193695, "ttft_relative": 1.678905, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9674.394671200012, "interactivity_tok_s": 98.61341724508944, "output_per_gpu": 48.19193337911507, "total_per_gpu": 429.4608491329223, "tpot_error_pct": 45.58, "tpot_ms": 10.14060792067112, "tpot_relative": 1.494111, "ttft_error_pct": 10.24, "ttft_ms": 199.3334394999992, "ttft_relative": 1.207534, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 6689.0810004497325, "interactivity_tok_s": 143.5654524681604, "output_per_gpu": 69.69203614779582, "total_per_gpu": 621.058316676821, "tpot_ms": 6.965464064007862, "tpot_relative": 1.026287, "ttft_ms": 180.81159114972252, "ttft_relative": 1.095331, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12896.602175050026, "interactivity_tok_s": 81.26394550640288, "output_per_gpu": 79.3232191017811, "total_per_gpu": null, "tpot_error_pct": 39.56, "tpot_ms": 12.305580214797827, "tpot_relative": 1.813097, "ttft_error_pct": 53.34, "ttft_ms": 307.9936153118481, "ttft_relative": 1.865782, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11285.403550800098, "interactivity_tok_s": 83.06886569898381, "output_per_gpu": 79.6870621810142, "total_per_gpu": 716.3391647303207, "tpot_error_pct": 36.52, "tpot_ms": 12.038204585863667, "tpot_relative": 1.773702, "ttft_error_pct": 9.81, "ttft_ms": 220.5630478500002, "ttft_relative": 1.33614, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 8306.297871901188, "interactivity_tok_s": 113.40803451964695, "output_per_gpu": 108.15698211196685, "total_per_gpu": 972.2667658376523, "tpot_ms": 8.817717406316206, "tpot_relative": 1.299197, "ttft_ms": 200.85074879752938, "ttft_relative": 1.216726, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17070.1487555503, "interactivity_tok_s": 61.14312501179821, "output_per_gpu": 119.85835796157019, "total_per_gpu": null, "tpot_error_pct": 52.73, "tpot_ms": 16.35506853480321, "tpot_relative": 2.409746, "ttft_error_pct": 31.0, "ttft_ms": 338.91364444661593, "ttft_relative": 2.053091, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15096.903396800082, "interactivity_tok_s": 62.46011687522548, "output_per_gpu": 120.85030997906924, "total_per_gpu": 1075.019381095859, "tpot_error_pct": 49.51, "tpot_ms": 16.010216599460854, "tpot_relative": 2.358936, "ttft_error_pct": 5.87, "ttft_ms": 273.91743990000015, "ttft_relative": 1.659353, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 10176.072957025235, "interactivity_tok_s": 93.3827354507831, "output_per_gpu": 179.06326951208592, "total_per_gpu": 1592.8505702734496, "tpot_ms": 10.70861755304914, "tpot_relative": 1.577801, "ttft_ms": 258.7209800124583, "ttft_relative": 1.567295, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22646.377975370528, "interactivity_tok_s": 45.92305881748765, "output_per_gpu": 180.69114648048034, "total_per_gpu": null, "tpot_error_pct": 51.42, "tpot_ms": 21.775552973818826, "tpot_relative": 3.208397, "ttft_error_pct": 12.88, "ttft_ms": 369.9872831538688, "ttft_relative": 2.241331, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20057.572082331226, "interactivity_tok_s": 46.43664640677444, "output_per_gpu": 177.94698664115094, "total_per_gpu": 1605.927014825523, "tpot_error_pct": 49.75, "tpot_ms": 21.534716164475526, "tpot_relative": 3.172912, "ttft_error_pct": 9.53, "ttft_ms": 359.01615089374934, "ttft_relative": 2.174869, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 13485.407603692147, "interactivity_tok_s": 69.53866887676662, "output_per_gpu": 264.1050903822937, "total_per_gpu": 2383.4823359676907, "tpot_ms": 14.38048809608588, "tpot_relative": 2.118813, "ttft_ms": 327.7658507984597, "ttft_relative": 1.985559, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31217.50026685997, "interactivity_tok_s": 33.19692192481122, "output_per_gpu": 262.16064483189933, "total_per_gpu": null, "tpot_error_pct": 51.62, "tpot_ms": 30.123274750138954, "tpot_relative": 4.438346, "ttft_error_pct": 16.06, "ttft_ms": 401.39019746782105, "ttft_relative": 2.431565, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28343.791364256333, "interactivity_tok_s": 33.26572994047807, "output_per_gpu": 255.66289932922257, "total_per_gpu": 2286.2190506550683, "tpot_error_pct": 51.31, "tpot_ms": 30.06096670024337, "tpot_relative": 4.429165, "ttft_error_pct": 10.87, "ttft_ms": 530.1945585312504, "ttft_relative": 3.211844, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 18865.0526388632, "interactivity_tok_s": 50.334254243151285, "output_per_gpu": 383.5438235638598, "total_per_gpu": 3429.7709933408228, "tpot_ms": 19.867186174434377, "tpot_relative": 2.92722, "ttft_ms": 478.19787642038136, "ttft_relative": 2.896855, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44353.07466356126, "interactivity_tok_s": 23.292573204067597, "output_per_gpu": 369.0386771189801, "total_per_gpu": null, "tpot_error_pct": 47.0, "tpot_ms": 42.93213940937059, "tpot_relative": 6.325596, "ttft_error_pct": 44.67, "ttft_ms": 433.4960477751428, "ttft_relative": 2.626058, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41151.363801759384, "interactivity_tok_s": 22.88589999981527, "output_per_gpu": 352.4297233199376, "total_per_gpu": 3176.7232781204993, "tpot_error_pct": 49.61, "tpot_ms": 43.6950261955209, "tpot_relative": 6.437999, "ttft_error_pct": 11.65, "ttft_ms": 874.7133784984368, "ttft_relative": 5.298891, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 27710.811555631335, "interactivity_tok_s": 34.23913198224503, "output_per_gpu": 523.1719683958818, "total_per_gpu": 4715.755966345018, "tpot_ms": 29.20634788634706, "tpot_relative": 4.303246, "ttft_ms": 783.4419463441009, "ttft_relative": 4.745981, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 60381.983697291565, "interactivity_tok_s": 17.0742209192388, "output_per_gpu": 542.1484687239312, "total_per_gpu": null, "tpot_error_pct": 30.72, "tpot_ms": 58.567826006820916, "tpot_relative": 8.629349, "ttft_error_pct": 66.75, "ttft_ms": 467.0976923137726, "ttft_relative": 2.829612, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 58067.893099487424, "interactivity_tok_s": 16.301724041907587, "output_per_gpu": 499.17673587259884, "total_per_gpu": 4506.608426661301, "tpot_error_pct": 36.91, "tpot_ms": 61.343205014957576, "tpot_relative": 9.038272, "ttft_error_pct": 14.3, "ttft_ms": 1605.7494168687506, "ttft_relative": 9.727404, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 42644.79162093776, "interactivity_tok_s": 22.318658714134738, "output_per_gpu": 680.2640398294377, "total_per_gpu": 6141.479427904337, "tpot_ms": 44.805559904309355, "tpot_relative": 6.601625, "ttft_ms": 1404.9038285516872, "ttft_relative": 8.51071, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 89751.7518750221, "interactivity_tok_s": 11.462457631202506, "output_per_gpu": 729.4787971511545, "total_per_gpu": null, "tpot_error_pct": 18.58, "tpot_ms": 87.24132574133596, "tpot_relative": 12.854086, "ttft_error_pct": 81.53, "ttft_ms": 503.8756416354322, "ttft_relative": 3.052408, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 91032.63677872045, "interactivity_tok_s": 10.497370746853848, "output_per_gpu": 639.8457978197783, "total_per_gpu": 5756.956491204192, "tpot_error_pct": 29.48, "tpot_ms": 95.26194931237505, "tpot_relative": 14.03584, "ttft_error_pct": 15.67, "ttft_ms": 3155.3597718238248, "ttft_relative": 19.114726, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 70597.32577288398, "interactivity_tok_s": 13.591724271912147, "output_per_gpu": 825.652547684303, "total_per_gpu": 7428.736439415121, "tpot_ms": 73.57418234760256, "tpot_relative": 10.840377, "ttft_ms": 2727.9692151101667, "ttft_relative": 16.525654, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 172945.54998217145, "interactivity_tok_s": 7.7501580550806555, "output_per_gpu": 668.3072353706845, "total_per_gpu": 6014.813670243343, "tpot_error_pct": 21.84, "tpot_ms": 129.02962660799477, "tpot_relative": 19.011149, "ttft_error_pct": 30.8, "ttft_ms": 53844.5181381244, "ttft_relative": 326.182518, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 138891.9105415059, "interactivity_tok_s": 9.443158201992205, "output_per_gpu": 836.5760395558248, "total_per_gpu": 7529.245132483768, "tpot_ms": 105.89677506292672, "tpot_relative": 15.602769, "ttft_ms": 41165.65799088337, "ttft_relative": 249.375766, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 52.41, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.23, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.87, "tpot_shape_error_pct": null, "ttft_mape_pct": 12.1, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "15b914a34e18ad72", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 9234.278169989397, "interactivity_tok_s": 113.66106509896757, "output_per_gpu": 13.847860942242638, "total_per_gpu": null, "tpot_error_pct": 52.41, "tpot_ms": 8.798087534454078, "tpot_relative": 1.524059, "ttft_error_pct": 53.23, "ttft_ms": 233.83462224287527, "ttft_relative": 1.53232, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8318.615716400005, "interactivity_tok_s": 114.05972975547213, "output_per_gpu": 13.979188841569064, "total_per_gpu": 125.13500268413473, "tpot_error_pct": 51.87, "tpot_ms": 8.767336220626316, "tpot_relative": 1.518732, "ttft_error_pct": 12.1, "ttft_ms": 171.0723611999994, "ttft_relative": 1.121039, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 5517.311888514087, "interactivity_tok_s": 173.22620788969243, "output_per_gpu": 21.075731665420314, "total_per_gpu": 188.65985490374524, "tpot_ms": 5.772798540026827, "tpot_relative": 1.0, "ttft_ms": 152.60164492065087, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 90.0, "points": 9, "status_counts": {"failed": 1, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.0, "tpot_shape_error_pct": 7.32, "ttft_mape_pct": 46.19, "ttft_shape_error_pct": 39.4}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.33, "tpot_shape_error_pct": 6.27, "ttft_mape_pct": 16.29, "ttft_shape_error_pct": 5.59}, "framework": "vllm", "id": "e505c65d55914cba", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9832.40989417612, "interactivity_tok_s": 106.81180827990056, "output_per_gpu": 26.010917237236463, "total_per_gpu": null, "tpot_error_pct": 44.4, "tpot_ms": 9.362260747234032, "tpot_relative": 1.44399, "ttft_error_pct": 49.87, "ttft_ms": 254.81714975570432, "ttft_relative": 1.498704, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8857.469695099997, "interactivity_tok_s": 107.21272906450886, "output_per_gpu": 26.257498812404837, "total_per_gpu": 235.04455241339625, "tpot_error_pct": 43.86, "tpot_ms": 9.327250679332206, "tpot_relative": 1.43859, "ttft_error_pct": 11.49, "ttft_ms": 189.55453319999933, "ttft_relative": 1.114863, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 6195.250885374844, "interactivity_tok_s": 154.2351373205334, "output_per_gpu": 37.53937084480799, "total_per_gpu": 336.0344669969075, "tpot_ms": 6.483606896409004, "tpot_relative": 1.0, "ttft_ms": 170.0250252848491, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10909.319419818155, "interactivity_tok_s": 96.30388355560834, "output_per_gpu": 46.88651787670601, "total_per_gpu": null, "tpot_error_pct": 48.58, "tpot_ms": 10.383797237237834, "tpot_relative": 1.601546, "ttft_error_pct": 54.33, "ttft_ms": 286.69484612385094, "ttft_relative": 1.686192, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9687.333280899988, "interactivity_tok_s": 98.5753852729021, "output_per_gpu": 48.12732165724969, "total_per_gpu": 428.8850638719751, "tpot_error_pct": 45.16, "tpot_ms": 10.144520330623504, "tpot_relative": 1.564641, "ttft_error_pct": 12.26, "ttft_ms": 208.54350954999913, "ttft_relative": 1.226546, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 6716.026250548748, "interactivity_tok_s": 143.090804354122, "output_per_gpu": 69.41560638959209, "total_per_gpu": 618.5949218644595, "tpot_ms": 6.988569283077016, "tpot_relative": 1.077883, "ttft_ms": 185.76544539973838, "ttft_relative": 1.092577, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12932.789432939791, "interactivity_tok_s": 81.09918927887841, "output_per_gpu": 79.10126468110745, "total_per_gpu": null, "tpot_error_pct": 51.3, "tpot_ms": 12.330579490274157, "tpot_relative": 1.901809, "ttft_error_pct": 57.16, "ttft_ms": 318.606614389327, "ttft_relative": 1.873881, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11312.964329949955, "interactivity_tok_s": 82.93730492774911, "output_per_gpu": 79.4960568003646, "total_per_gpu": 714.6221402712846, "tpot_error_pct": 47.94, "tpot_ms": 12.057300401443605, "tpot_relative": 1.859659, "ttft_error_pct": 13.73, "ttft_ms": 230.56016122500037, "ttft_relative": 1.356037, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 7696.511735925014, "interactivity_tok_s": 122.70086836546946, "output_per_gpu": 116.8812117742624, "total_per_gpu": 1050.6923874901124, "tpot_ms": 8.149901572183335, "tpot_relative": 1.257001, "ttft_ms": 202.72917004913324, "ttft_relative": 1.192349, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16777.621954308543, "interactivity_tok_s": 62.27540924877753, "output_per_gpu": 121.94815245998443, "total_per_gpu": null, "tpot_error_pct": 60.65, "tpot_ms": 16.057702583779168, "tpot_relative": 2.476662, "ttft_error_pct": 33.67, "ttft_ms": 350.5922111024527, "ttft_relative": 2.062004, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14833.599234700068, "interactivity_tok_s": 63.643056176248784, "output_per_gpu": 122.97103542232504, "total_per_gpu": 1093.884214408061, "tpot_error_pct": 57.2, "tpot_ms": 15.71263324046959, "tpot_relative": 2.42344, "ttft_error_pct": 8.91, "ttft_ms": 285.6598557, "ttft_relative": 1.680105, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 9517.80673327512, "interactivity_tok_s": 100.04661657101127, "output_per_gpu": 191.4487271908037, "total_per_gpu": 1703.0249426078706, "tpot_ms": 9.99534051499101, "tpot_relative": 1.541633, "ttft_ms": 262.29164072510684, "ttft_relative": 1.542665, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22263.465923208212, "interactivity_tok_s": 46.753469526954596, "output_per_gpu": 183.7988754362975, "total_per_gpu": null, "tpot_error_pct": 57.77, "tpot_ms": 21.38878697384103, "tpot_relative": 3.298902, "ttft_error_pct": 12.79, "ttft_ms": 382.7368489688398, "ttft_relative": 2.251062, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19719.355769975, "interactivity_tok_s": 47.28504973333112, "output_per_gpu": 180.96089385527603, "total_per_gpu": 1633.1267730608315, "tpot_error_pct": 55.99, "tpot_ms": 21.148333471987495, "tpot_relative": 3.261816, "ttft_error_pct": 10.05, "ttft_ms": 373.4228448624999, "ttft_relative": 2.196282, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 12743.215236145625, "interactivity_tok_s": 73.76203322970075, "output_per_gpu": 279.8988146284883, "total_per_gpu": 2526.0167441665612, "tpot_ms": 13.557110022793456, "tpot_relative": 2.090983, "ttft_ms": 339.3280418611539, "ttft_relative": 1.995754, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30742.038008371976, "interactivity_tok_s": 33.732522835983524, "output_per_gpu": 266.2152716671306, "total_per_gpu": null, "tpot_error_pct": 51.63, "tpot_ms": 29.644981042842996, "tpot_relative": 4.572298, "ttft_error_pct": 15.48, "ttft_ms": 415.2224015435899, "ttft_relative": 2.442125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27916.446835571805, "interactivity_tok_s": 33.81202214252367, "output_per_gpu": 259.5799401766185, "total_per_gpu": 2321.246477125652, "tpot_error_pct": 51.28, "tpot_ms": 29.57527934250198, "tpot_relative": 4.561547, "ttft_error_pct": 11.99, "ttft_ms": 550.2186470874994, "ttft_relative": 3.236104, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 18583.200362621574, "interactivity_tok_s": 51.15004483410891, "output_per_gpu": 389.9202349225602, "total_per_gpu": 3486.790894004243, "tpot_ms": 19.550324994694037, "tpot_relative": 3.015347, "ttft_ms": 491.29197684378596, "ttft_relative": 2.889527, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44222.670104299716, "interactivity_tok_s": 23.369911876093884, "output_per_gpu": 370.1269046259728, "total_per_gpu": null, "tpot_error_pct": 45.93, "tpot_ms": 42.790062936563494, "tpot_relative": 6.599731, "ttft_error_pct": 44.44, "ttft_ms": 448.43572019525845, "ttft_relative": 2.637469, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41050.12493251254, "interactivity_tok_s": 22.96314390721164, "output_per_gpu": 353.4007147248211, "total_per_gpu": 3185.4755790606455, "tpot_error_pct": 48.52, "tpot_ms": 43.54804394558304, "tpot_relative": 6.716639, "ttft_error_pct": 12.3, "ttft_ms": 906.4610678187495, "ttft_relative": 5.331339, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 27839.896885667134, "interactivity_tok_s": 34.104454679459444, "output_per_gpu": 521.4047563179005, "total_per_gpu": 4699.826709037732, "tpot_ms": 29.32168273613487, "tpot_relative": 4.522434, "ttft_ms": 807.1647671813821, "ttft_relative": 4.747329, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 62238.343464238234, "interactivity_tok_s": 16.565421158576005, "output_per_gpu": 525.9780093409764, "total_per_gpu": null, "tpot_error_pct": 33.3, "tpot_ms": 60.36671150267102, "tpot_relative": 9.310668, "ttft_error_pct": 66.67, "ttft_ms": 483.19759700577674, "ttft_relative": 2.84192, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 59876.499361533926, "interactivity_tok_s": 15.811659880212089, "output_per_gpu": 484.44520536641033, "total_per_gpu": 4373.610963546852, "tpot_error_pct": 39.66, "tpot_ms": 63.24446690454529, "tpot_relative": 9.754519, "ttft_error_pct": 14.75, "ttft_ms": 1663.6701453898447, "ttft_relative": 9.784855, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 43135.83910012012, "interactivity_tok_s": 22.08239337648556, "output_per_gpu": 673.3143599613396, "total_per_gpu": 6078.737149257437, "tpot_ms": 45.28494638017137, "tpot_relative": 6.98453, "ttft_ms": 1449.8455428281886, "ttft_relative": 8.527248, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 93302.51337895392, "interactivity_tok_s": 11.025932772772647, "output_per_gpu": 701.7174310629921, "total_per_gpu": null, "tpot_error_pct": 20.43, "tpot_ms": 90.69527455032124, "tpot_relative": 13.988398, "ttft_error_pct": 81.28, "ttft_ms": 521.2475139752955, "ttft_relative": 3.065711, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 94728.57695607483, "interactivity_tok_s": 10.086300714880007, "output_per_gpu": 615.2549611597531, "total_per_gpu": 5535.7025934424855, "tpot_error_pct": 31.65, "tpot_ms": 99.14437693937985, "tpot_relative": 15.291547, "ttft_error_pct": 17.42, "ttft_ms": 3269.3150803980457, "ttft_relative": 19.228435, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 72254.49279160833, "interactivity_tok_s": 13.27861585251875, "output_per_gpu": 807.3714126099379, "total_per_gpu": 7264.253528702014, "tpot_ms": 75.30905412933649, "tpot_relative": 11.615302, "ttft_ms": 2784.31066516614, "ttft_relative": 16.375887, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 181488.5354432089, "interactivity_tok_s": 7.683380496633486, "output_per_gpu": 639.5718515798889, "total_per_gpu": 5756.1931285269275, "tpot_error_pct": 22.05, "tpot_ms": 130.15104489985305, "tpot_relative": 20.073864, "ttft_error_pct": 50.03, "ttft_ms": 61350.535223819476, "ttft_relative": 360.832384, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31148296509", "measured": {"e2e_ms": 139299.8636483847, "interactivity_tok_s": 9.377920021018685, "output_per_gpu": 834.0572257288424, "total_per_gpu": 7506.575625051414, "tpot_ms": 106.63345366122819, "tpot_relative": 16.446625, "ttft_ms": 40891.245564054065, "ttft_relative": 240.501335, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 20.59, "points": 7, "status_counts": {"failed": 27, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 80.68, "tpot_shape_error_pct": 17.76, "ttft_mape_pct": 52.02, "ttft_shape_error_pct": 36.72}, "aisimulate": {"coverage_pct": 55.88, "points": 19, "status_counts": {"failed": 15, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.69, "tpot_shape_error_pct": 9.43, "ttft_mape_pct": 45.83, "ttft_shape_error_pct": 52.35}, "gpu": "gb200", "precisions": ["fp4"], "rows": 34, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "3e03088ab74db2c6", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 10104.001975167193, "interactivity_tok_s": 180.2668913092951, "output_per_gpu": 21.716008088396766, "total_per_gpu": 171.81440585282542, "tpot_ms": 5.547330365198554, "tpot_relative": 1.0, "ttft_ms": 4973.4160695792525, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 9187.028458163451, "interactivity_tok_s": 149.2691448311982, "output_per_gpu": 71.47663757299023, "total_per_gpu": 572.024848642656, "tpot_ms": 6.6993081599740885, "tpot_relative": 1.207663, "ttft_ms": 3038.089487387333, "ttft_relative": 0.610866, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "259133e2f2fc0fac", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 156, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 17233.27648133611, "interactivity_tok_s": 66.80682818803284, "output_per_gpu": 500.2117658896651, "total_per_gpu": 3609.6676099688857, "tpot_ms": 14.968529821314446, "tpot_relative": 1.0, "ttft_ms": 3469.4784700002438, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "43d3f690eaa8969c", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 43286.70943141382, "interactivity_tok_s": 23.347931160986338, "output_per_gpu": 5520.37875670067, "total_per_gpu": 14187.710050759697, "tpot_ms": 42.83034728451524, "tpot_relative": 1.0, "ttft_ms": 3824.3061084318847, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "c08aaf7e339be1d9", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 27097.10373138249, "interactivity_tok_s": 39.15676961082336, "output_per_gpu": 4604.819596179517, "total_per_gpu": 15061.864123731299, "tpot_ms": 25.538368203989663, "tpot_relative": 1.0, "ttft_ms": 3565.5803592204134, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "40b0343ec3d97b42", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 666, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 16409.574612458488, "interactivity_tok_s": 74.78341526152302, "output_per_gpu": 1112.2644730184347, "total_per_gpu": 6678.8785324540295, "tpot_ms": 13.371948800451644, "tpot_relative": 1.0, "ttft_ms": 4099.659986570184, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "7da3133d992473d7", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 5940.389176455792, "interactivity_tok_s": 165.2137665260803, "output_per_gpu": 38.32387515642297, "total_per_gpu": 284.85997966368205, "tpot_ms": 6.052764373253013, "tpot_relative": 1.0, "ttft_ms": 353.95356675377116, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 30, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 10514.109520881515, "interactivity_tok_s": 130.87453537878903, "output_per_gpu": 125.16407288834758, "total_per_gpu": 936.3673622520754, "tpot_ms": 7.640905827139776, "tpot_relative": 1.262383, "ttft_ms": 3471.9468308791206, "ttft_relative": 9.809046, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 12853.449286856532, "interactivity_tok_s": 107.55556157125025, "output_per_gpu": 205.5833895772142, "total_per_gpu": 1542.4000857991268, "tpot_ms": 9.297520141137001, "tpot_relative": 1.536078, "ttft_ms": 4281.454560548106, "ttft_relative": 12.096091, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 105, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 14359.387138231014, "interactivity_tok_s": 85.26237285085027, "output_per_gpu": 322.8593890515256, "total_per_gpu": 2424.914523052687, "tpot_ms": 11.72850304963132, "tpot_relative": 1.93771, "ttft_ms": 3565.347406088175, "ttft_relative": 10.072924, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "838e33585cd7c21d", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 615, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24610706849", "measured": {"e2e_ms": 23323.748798574, "interactivity_tok_s": 48.69708468625244, "output_per_gpu": 1451.7018627278412, "total_per_gpu": 7470.288816009917, "tpot_ms": 20.53511019074018, "tpot_relative": 1.0, "ttft_ms": 4415.656198462194, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "ec8b34a468892df1", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 333, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 15147.31378455803, "interactivity_tok_s": 81.20900577288815, "output_per_gpu": 602.4717489640768, "total_per_gpu": 4333.688387627042, "tpot_ms": 12.313905218795977, "tpot_relative": 1.0, "ttft_ms": 3798.3845992453603, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "365bf0e2504451ef", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 18386.25140104138, "interactivity_tok_s": 62.26679167962809, "output_per_gpu": 1842.2968552382206, "total_per_gpu": 9472.611046082004, "tpot_ms": 16.05992492989118, "tpot_relative": 1.0, "ttft_ms": 3594.608498764733, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "97bb0ca6db1011f7", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 23052.084137528953, "interactivity_tok_s": 50.01811160958451, "output_per_gpu": 2940.33314232467, "total_per_gpu": 13228.624425242879, "tpot_ms": 19.99275797945917, "tpot_relative": 1.0, "ttft_ms": 4638.361448985007, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "3808685a93f057dd", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb200: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4301, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27591355916", "measured": {"e2e_ms": 39565.57521765737, "interactivity_tok_s": 25.814694921248673, "output_per_gpu": 6048.66725862637, "total_per_gpu": 16760.079342115892, "tpot_ms": 38.737626109881965, "tpot_relative": 1.0, "ttft_ms": 3912.7424452586993, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 52}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.65, "tpot_shape_error_pct": 17.04, "ttft_mape_pct": 52.13, "ttft_shape_error_pct": 49.4}, "framework": "dynamo-vllm", "id": "ce174413d9686120", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26177.816621838312, "interactivity_tok_s": 37.20165039493554, "output_per_gpu": 618.1437486267102, "total_per_gpu": 5563.3386452163095, "tpot_error_pct": 31.45, "tpot_ms": 26.880527863251338, "tpot_relative": 1.314517, "ttft_error_pct": 36.43, "ttft_ms": 1418.6079165183648, "ttft_relative": 0.635677, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 21015.455781842502, "interactivity_tok_s": 48.90218601668455, "output_per_gpu": 1335.1660913664384, "total_per_gpu": 6884.776693882756, "tpot_ms": 20.448983602876524, "tpot_relative": 1.0, "ttft_ms": 2231.6480318024333, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 35163.68477150864, "interactivity_tok_s": 30.27479681104434, "output_per_gpu": 923.0169124232378, "total_per_gpu": 8305.448540848389, "tpot_error_pct": 53.85, "tpot_ms": 33.03077494595098, "tpot_relative": 1.615277, "ttft_error_pct": 67.83, "ttft_ms": 4748.443806703711, "ttft_relative": 2.127775, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 34461.0540645539, "interactivity_tok_s": 46.578587661953144, "output_per_gpu": 1466.3288449509475, "total_per_gpu": 7565.244248446832, "tpot_ms": 21.46909234899004, "tpot_relative": 1.049886, "ttft_ms": 14762.361920763033, "ttft_relative": 6.615005, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 118.72, "tpot_shape_error_pct": null, "ttft_mape_pct": 95.66, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 75.26, "tpot_shape_error_pct": null, "ttft_mape_pct": 65.93, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "a918651bd8bf99af", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 61894.861, "interactivity_tok_s": 16.738642830839275, "output_per_gpu": 1183.841275, "total_per_gpu": null, "tpot_error_pct": 118.72, "tpot_ms": 59.742, "tpot_relative": 2.18717, "ttft_error_pct": 95.66, "ttft_ms": 778.795, "ttft_relative": 0.043442, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 50179.440160569524, "interactivity_tok_s": 20.888875552975097, "output_per_gpu": 1364.5517479447394, "total_per_gpu": 12281.621211053545, "tpot_error_pct": 75.26, "tpot_ms": 47.872370988278256, "tpot_relative": 1.75262, "ttft_error_pct": 65.93, "ttft_ms": 6107.3718710283365, "ttft_relative": 0.340675, "unavailable_metrics": {}}, "concurrency": 3072, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 43078.87345832933, "interactivity_tok_s": 36.61025538290807, "output_per_gpu": 3731.597272087916, "total_per_gpu": 13437.247342416173, "tpot_ms": 27.31475073148115, "tpot_relative": 1.0, "ttft_ms": 17927.25443928227, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 86.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 93.34, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.57, "tpot_shape_error_pct": null, "ttft_mape_pct": 38.89, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "610ad820cb373fcf", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 25452.532, "interactivity_tok_s": 41.461088768191054, "output_per_gpu": 244.37555000000003, "total_per_gpu": null, "tpot_error_pct": 86.33, "tpot_ms": 24.119, "tpot_relative": 1.863252, "ttft_error_pct": 93.34, "ttft_ms": 778.795, "ttft_relative": 0.066572, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32851.771581057, "interactivity_tok_s": 55.35160122171674, "output_per_gpu": 171.8564304815068, "total_per_gpu": 1551.5339200053543, "tpot_error_pct": 39.57, "tpot_ms": 18.066324693921562, "tpot_relative": 1.395668, "ttft_error_pct": 38.89, "ttft_ms": 16248.078093506203, "ttft_relative": 1.388899, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "27885061956", "measured": {"e2e_ms": 23596.59381914189, "interactivity_tok_s": 77.25246346065482, "output_per_gpu": 298.45720488449314, "total_per_gpu": 2156.1349473705122, "tpot_ms": 12.944571023412172, "tpot_relative": 1.0, "ttft_ms": 11698.527754688075, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.99, "tpot_shape_error_pct": 14.02, "ttft_mape_pct": 29.74, "ttft_shape_error_pct": 25.91}, "framework": "dynamo-vllm", "id": "4c5ca5cd71f64363", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8621.328229599976, "interactivity_tok_s": 111.65200124525607, "output_per_gpu": 20.728023968580697, "total_per_gpu": 186.33257356676447, "tpot_error_pct": 25.87, "tpot_ms": 8.95640014372325, "tpot_relative": 1.258669, "ttft_error_pct": 17.71, "ttft_ms": 389.2620468499974, "ttft_relative": 0.82289, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 7012.906322954223, "interactivity_tok_s": 140.53290677734117, "output_per_gpu": 32.06358572691131, "total_per_gpu": 230.72539867567178, "tpot_ms": 7.115771123872, "tpot_relative": 1.0, "ttft_ms": 473.0423733126372, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9465.727676162518, "interactivity_tok_s": 102.79473403815489, "output_per_gpu": 38.207135718025306, "total_per_gpu": 339.87013686726, "tpot_error_pct": 25.87, "tpot_ms": 9.728124785349651, "tpot_relative": 1.367122, "ttft_error_pct": 5.86, "ttft_ms": 467.38763716250094, "ttft_relative": 0.988046, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 7644.945074256975, "interactivity_tok_s": 129.38604412959435, "output_per_gpu": 59.0966154404346, "total_per_gpu": 420.80868848602074, "tpot_ms": 7.728808827313633, "tpot_relative": 1.086152, "ttft_ms": 496.4838879648596, "ttft_relative": 1.049555, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10816.754080487504, "interactivity_tok_s": 88.80271722582324, "output_per_gpu": 65.22927789195239, "total_per_gpu": 588.6779062771886, "tpot_error_pct": 29.65, "tpot_ms": 11.260916684080998, "tpot_relative": 1.582529, "ttft_error_pct": 17.0, "ttft_ms": 522.0592150562466, "ttft_relative": 1.10362, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 8540.238440479152, "interactivity_tok_s": 115.13142874563779, "output_per_gpu": 102.63066955824245, "total_per_gpu": 743.8392578597934, "tpot_ms": 8.685725617192855, "tpot_relative": 1.22063, "ttft_ms": 628.9560385746881, "ttft_relative": 1.329598, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13832.816020906259, "interactivity_tok_s": 70.1699906561441, "output_per_gpu": 103.39752356079484, "total_per_gpu": 924.6135781744451, "tpot_error_pct": 38.23, "tpot_ms": 14.251106358276818, "tpot_relative": 2.002749, "ttft_error_pct": 18.64, "ttft_ms": 660.247781390624, "ttft_relative": 1.395748, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 10306.08648394118, "interactivity_tok_s": 96.99318080774928, "output_per_gpu": 171.20204301897022, "total_per_gpu": 1229.6232293392923, "tpot_ms": 10.310003153542366, "tpot_relative": 1.448895, "ttft_ms": 811.5447501186281, "ttft_relative": 1.715586, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22428.550579772662, "interactivity_tok_s": 43.333863688731824, "output_per_gpu": 254.23489672789688, "total_per_gpu": 2295.253455557119, "tpot_error_pct": 80.32, "tpot_ms": 23.076640642593603, "tpot_relative": 3.243027, "ttft_error_pct": 89.5, "ttft_ms": 1221.7145098703136, "ttft_relative": 2.582675, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 23374.956036536605, "interactivity_tok_s": 78.1385217460957, "output_per_gpu": 300.2509486404275, "total_per_gpu": 2173.051483033971, "tpot_ms": 12.7977849804916, "tpot_relative": 1.79851, "ttft_ms": 11635.647241672268, "ttft_relative": 24.597473, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 71.95, "tpot_shape_error_pct": 17.76, "ttft_mape_pct": 35.02, "ttft_shape_error_pct": 36.72}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.69, "tpot_shape_error_pct": 3.11, "ttft_mape_pct": 32.85, "ttft_shape_error_pct": 57.36}, "framework": "dynamo-vllm", "id": "74551d7f8b94354e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10613.917, "interactivity_tok_s": 104.01497815685458, "output_per_gpu": 10.676916666666667, "total_per_gpu": null, "tpot_error_pct": 54.24, "tpot_ms": 9.614, "tpot_relative": 1.542355, "ttft_error_pct": 17.23, "ttft_ms": 778.795, "ttft_relative": 1.172304, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8561.468274049994, "interactivity_tok_s": 112.4609700371255, "output_per_gpu": 11.59573587131735, "total_per_gpu": 104.23874994442879, "tpot_error_pct": 42.65, "tpot_ms": 8.891973808067643, "tpot_relative": 1.426521, "ttft_error_pct": 41.5, "ttft_ms": 388.6229677750006, "ttft_relative": 0.584986, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 6395.03549055662, "interactivity_tok_s": 160.42797472251502, "output_per_gpu": 17.347445293365773, "total_per_gpu": 138.69995928319415, "tpot_ms": 6.233326835482743, "tpot_relative": 1.0, "ttft_ms": 664.3283340905327, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11124.394, "interactivity_tok_s": 98.88262632255513, "output_per_gpu": 20.202388888888887, "total_per_gpu": null, "tpot_error_pct": 55.8, "tpot_ms": 10.113, "tpot_relative": 1.622408, "ttft_error_pct": 21.09, "ttft_ms": 778.795, "ttft_relative": 1.172304, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9073.984955187445, "interactivity_tok_s": 107.53160887617528, "output_per_gpu": 22.15590342388676, "total_per_gpu": 197.08700449747752, "tpot_error_pct": 43.27, "tpot_ms": 9.299591166272972, "tpot_relative": 1.491915, "ttft_error_pct": 26.53, "ttft_ms": 472.54543586249713, "ttft_relative": 0.711313, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 6646.3837361719925, "interactivity_tok_s": 154.05699098872037, "output_per_gpu": 34.264605977353554, "total_per_gpu": 271.09738096504844, "tpot_ms": 6.49110432173258, "tpot_relative": 1.041355, "ttft_ms": 643.1627323559951, "ttft_relative": 0.96814, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11552.008, "interactivity_tok_s": 94.95774380400721, "output_per_gpu": 38.83463888888889, "total_per_gpu": null, "tpot_error_pct": 43.75, "tpot_ms": 10.531, "tpot_relative": 1.689467, "ttft_error_pct": 10.07, "ttft_ms": 778.795, "ttft_relative": 1.172304, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9424.047960586133, "interactivity_tok_s": 103.21330753867053, "output_per_gpu": 41.52143623788558, "total_per_gpu": 374.7205693527366, "tpot_error_pct": 32.25, "tpot_ms": 9.688673135732365, "tpot_relative": 1.554334, "ttft_error_pct": 19.88, "ttft_ms": 566.8488890875019, "ttft_relative": 0.853266, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 7405.151018753531, "interactivity_tok_s": 136.50370367680145, "output_per_gpu": 59.256712015913685, "total_per_gpu": 475.6451056308336, "tpot_ms": 7.3258085536469455, "tpot_relative": 1.175265, "ttft_ms": 707.539589049702, "ttft_relative": 1.065045, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14043.013, "interactivity_tok_s": 77.12478790683326, "output_per_gpu": 63.11936111111111, "total_per_gpu": null, "tpot_error_pct": 55.29, "tpot_ms": 12.966, "tpot_relative": 2.080109, "ttft_error_pct": 29.61, "ttft_ms": 778.795, "ttft_relative": 1.172304, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11512.393634403088, "interactivity_tok_s": 86.00174619934442, "output_per_gpu": 68.81641741248589, "total_per_gpu": 615.3783161304916, "tpot_error_pct": 39.26, "tpot_ms": 11.627670881031749, "tpot_relative": 1.865404, "ttft_error_pct": 30.84, "ttft_ms": 765.2072113031248, "ttft_relative": 1.151851, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 8824.425267110928, "interactivity_tok_s": 119.77015528351254, "output_per_gpu": 101.19025214423966, "total_per_gpu": 804.8197018424416, "tpot_ms": 8.349325402750473, "tpot_relative": 1.339465, "ttft_ms": 1106.4670252621, "ttft_relative": 1.665542, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26682.178, "interactivity_tok_s": 39.49291102247146, "output_per_gpu": 236.67199999999997, "total_per_gpu": null, "tpot_error_pct": 150.67, "tpot_ms": 25.321, "tpot_relative": 4.062197, "ttft_error_pct": 97.11, "ttft_ms": 778.795, "ttft_relative": 1.172304, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 52838.07153303273, "interactivity_tok_s": 67.79954040973917, "output_per_gpu": 118.40293741477785, "total_per_gpu": 1065.3200528162924, "tpot_error_pct": 46.02, "tpot_ms": 14.749362517158795, "tpot_relative": 2.36621, "ttft_error_pct": 45.48, "ttft_ms": 39253.27849831621, "ttft_relative": 59.087166, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 36285.54366946064, "interactivity_tok_s": 98.99798473292364, "output_per_gpu": 193.55531277937828, "total_per_gpu": 1548.3081575037463, "tpot_ms": 10.101215723711912, "tpot_relative": 1.620518, "ttft_ms": 26982.465586524086, "ttft_relative": 40.61616, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 68.09, "tpot_shape_error_pct": null, "ttft_mape_pct": 40.09, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e4083514468fa502", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 74662.19925326783, "interactivity_tok_s": 14.004933589791465, "output_per_gpu": 868.3780272356171, "total_per_gpu": 7808.309205238582, "tpot_error_pct": 68.09, "tpot_ms": 71.40340891933427, "tpot_relative": 1.680931, "ttft_error_pct": 40.09, "ttft_ms": 8944.417415601707, "ttft_relative": 1.400948, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 45414.047950715256, "interactivity_tok_s": 23.541320505079025, "output_per_gpu": 4885.05414293751, "total_per_gpu": 12586.663781137877, "tpot_ms": 42.47850071894865, "tpot_relative": 1.0, "ttft_ms": 6384.547383994641, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 126.78, "tpot_shape_error_pct": 8.74, "ttft_mape_pct": 70.89, "ttft_shape_error_pct": 141.0}, "framework": "dynamo-vllm", "id": "1d692d88c2b1506b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 58692.19747952374, "interactivity_tok_s": 16.776960690924554, "output_per_gpu": 1170.469826656671, "total_per_gpu": 10534.790689775216, "tpot_error_pct": 117.28, "tpot_ms": 59.60555182923848, "tpot_relative": 2.17279, "ttft_error_pct": 82.93, "ttft_ms": 3818.2639817069226, "ttft_relative": 0.17073, "unavailable_metrics": {}}, "concurrency": 3072, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 47540.06404016961, "interactivity_tok_s": 36.452809009386655, "output_per_gpu": 3444.3199453840534, "total_per_gpu": 12438.81771134897, "tpot_ms": 27.43272815388516, "tpot_relative": 1.0, "ttft_ms": 22364.379770904936, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 75014.39998928968, "interactivity_tok_s": 15.464370380446693, "output_per_gpu": 1214.1136511087557, "total_per_gpu": 10929.419772678844, "tpot_error_pct": 136.28, "tpot_ms": 64.6647729845122, "tpot_relative": 2.357213, "ttft_error_pct": 58.85, "ttft_ms": 15506.29933795522, "ttft_relative": 0.693348, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24100518225", "measured": {"e2e_ms": 62795.43750858545, "interactivity_tok_s": 36.53854102731838, "output_per_gpu": 3481.597922731994, "total_per_gpu": 12576.263311761015, "tpot_ms": 27.368361513185235, "tpot_relative": 0.997654, "ttft_ms": 37686.43457658645, "ttft_relative": 1.68511, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.87, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.61, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "228ed86e97edf8d5", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 35163.68477150864, "interactivity_tok_s": 30.27479681104434, "output_per_gpu": 923.0169124232378, "total_per_gpu": 8305.448540848389, "tpot_error_pct": 44.87, "tpot_ms": 33.03077494595098, "tpot_relative": 1.448725, "ttft_error_pct": 41.61, "ttft_ms": 4748.443806703711, "ttft_relative": 0.583887, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 29124.810146198477, "interactivity_tok_s": 43.85986906793629, "output_per_gpu": 1916.700489054777, "total_per_gpu": 9858.37256505851, "tpot_ms": 22.799885664297364, "tpot_relative": 1.0, "ttft_ms": 8132.469371811066, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.15, "tpot_shape_error_pct": null, "ttft_mape_pct": 125.25, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "0fc424036c62df7f", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 121417.3095941544, "interactivity_tok_s": 25.838841787400952, "output_per_gpu": 893.2561207201962, "total_per_gpu": 8041.294016789708, "tpot_error_pct": 12.15, "tpot_ms": 38.70142509590353, "tpot_relative": 1.121523, "ttft_error_pct": 125.25, "ttft_ms": 85792.9144496123, "ttft_relative": 2.252479, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27885061956", "measured": {"e2e_ms": 69857.71247809404, "interactivity_tok_s": 28.97885948657033, "output_per_gpu": 4405.2230206349295, "total_per_gpu": 13222.27010674368, "tpot_ms": 34.50791431123885, "tpot_relative": 1.0, "ttft_ms": 38088.22574244914, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}]}, {"aic": {"coverage_pct": 16.67, "points": 3, "status_counts": {"failed": 15, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 59.94, "tpot_shape_error_pct": 5.32, "ttft_mape_pct": 37.83, "ttft_shape_error_pct": 62.23}, "aisimulate": {"coverage_pct": 27.78, "points": 5, "status_counts": {"failed": 13, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 37.42, "tpot_shape_error_pct": 4.07, "ttft_mape_pct": 20.05, "ttft_shape_error_pct": 33.38}, "gpu": "gb300", "precisions": ["fp4"], "rows": 18, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "53203c2a464dd07e", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 615, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 15288.12294987371, "interactivity_tok_s": 74.9220112197133, "output_per_gpu": 1103.1523673805289, "total_per_gpu": 7224.882868090179, "tpot_ms": 13.347212437576454, "tpot_relative": 1.0, "ttft_ms": 2998.010440577745, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "b787b9ca9e8e491c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 5312.901916125702, "interactivity_tok_s": 182.34794960472053, "output_per_gpu": 21.048616908372637, "total_per_gpu": 168.29234845751762, "tpot_ms": 5.484021082593586, "tpot_relative": 1.0, "ttft_ms": 272.41011002624873, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 12, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 8141.518592196128, "interactivity_tok_s": 167.643471090583, "output_per_gpu": 40.261456532226866, "total_per_gpu": 323.32979620520325, "tpot_ms": 5.965039935612337, "tpot_relative": 1.087713, "ttft_ms": 2684.656244467866, "ttft_relative": 9.8552, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 9069.373763808591, "interactivity_tok_s": 150.81626534233038, "output_per_gpu": 72.96638043018754, "total_per_gpu": 583.9472048326007, "tpot_ms": 6.630584557508764, "tpot_relative": 1.209073, "ttft_ms": 2983.2796349617033, "ttft_relative": 10.951428, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "487774e158574788", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 30, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 10374.37396703676, "interactivity_tok_s": 132.49355134330654, "output_per_gpu": 127.54048332342583, "total_per_gpu": 954.1455722397571, "tpot_ms": 7.547537143214473, "tpot_relative": 1.0, "ttft_ms": 3417.6894238994764, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 60, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 12784.742845255727, "interactivity_tok_s": 107.96342225905596, "output_per_gpu": 206.99403831490054, "total_per_gpu": 1552.9835514113747, "tpot_ms": 9.26239627343899, "tpot_relative": 1.227208, "ttft_ms": 4245.365881656374, "ttft_relative": 1.242174, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 115, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 15603.363109965727, "interactivity_tok_s": 85.26089232188994, "output_per_gpu": 322.06715369944106, "total_per_gpu": 2425.4578557957784, "tpot_ms": 11.728706711450394, "tpot_relative": 1.553978, "ttft_ms": 4840.113946095555, "ttft_relative": 1.416195, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "dc5cc9224d81220c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1229, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 18186.50672119952, "interactivity_tok_s": 63.03613701480573, "output_per_gpu": 1861.2251163921167, "total_per_gpu": 10306.084104203743, "tpot_ms": 15.8639162765498, "tpot_relative": 1.0, "ttft_ms": 3575.5118982449053, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 52}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "3d790164268695f9", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 333, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 15099.974048308548, "interactivity_tok_s": 81.13253259949818, "output_per_gpu": 604.1286439237954, "total_per_gpu": 4345.606733107635, "tpot_ms": 12.325511948904516, "tpot_relative": 1.0, "ttft_ms": 3740.20139782098, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "eb0c64e84713a39c", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2151, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 20499.780423378877, "interactivity_tok_s": 49.511819810104896, "output_per_gpu": 2903.5708098915247, "total_per_gpu": 13059.119584327153, "tpot_ms": 20.197197433569375, "tpot_relative": 1.0, "ttft_ms": 1890.5241023869155, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "ac67ba3200710956", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: unsupported model/backend/GPU for KV-cache estimation: model=[path], backend=trtllm, gpu_sku=gb300: Kimi model-level vision exclusions require supported modelopt matching semantics", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2253, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27663808752", "measured": {"e2e_ms": 27035.31826611308, "interactivity_tok_s": 39.1903078579325, "output_per_gpu": 4615.404003561145, "total_per_gpu": 16606.133026141393, "tpot_ms": 25.51651300176225, "tpot_relative": 1.0, "ttft_ms": 3524.155491531481, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 1.88, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "29a8457abca6c7c3", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 69092.5831041512, "interactivity_tok_s": 20.356974930772793, "output_per_gpu": 1640.6638765450869, "total_per_gpu": 14766.763002381565, "tpot_error_pct": 18.59, "tpot_ms": 49.12321223564222, "tpot_relative": 1.185919, "ttft_error_pct": 1.88, "ttft_ms": 23866.882520493004, "ttft_relative": 1.018845, "unavailable_metrics": {}}, "concurrency": 3072, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 61562.740478730186, "interactivity_tok_s": 24.141714644264788, "output_per_gpu": 5499.905289054974, "total_per_gpu": 16504.01118257683, "tpot_ms": 41.42207853647895, "tpot_relative": 1.0, "ttft_ms": 23425.43875632091, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.38, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1a89f59a26dad813", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 54427.95327498603, "interactivity_tok_s": 34.05747402428457, "output_per_gpu": 1182.6607450984761, "total_per_gpu": 10634.286558382864, "tpot_error_pct": 32.05, "tpot_ms": 29.362130593917602, "tpot_relative": 1.320514, "ttft_error_pct": 16.38, "ttft_ms": 27371.95620872915, "ttft_relative": 0.836244, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 53220.55435023458, "interactivity_tok_s": 44.97335825856764, "output_per_gpu": 2112.7904830265725, "total_per_gpu": 10859.795238616587, "tpot_ms": 22.23538643146568, "tpot_relative": 1.0, "ttft_ms": 32732.006185230526, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 28}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1ccae9b3cbc5e938", "is_multinode": true, "parallelism": {"attention_dp_size": 24, "moe_ep_size": 24, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external decode prediction failed: AIC static_gen evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"nvfp4\", distribution: \"power_law_1.01\", topk: 8, num_experts: 384, hidden_size: 7168, inter_size: 2048, moe_tp_size: 1, moe_ep_size: 24 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 78290.26460913666, "interactivity_tok_s": 54.45159160201041, "output_per_gpu": 955.7345777716586, "total_per_gpu": 6447.655361162307, "tpot_ms": 18.36493609422941, "tpot_relative": 1.0, "ttft_ms": 61368.06174588037, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 63.4, "tpot_shape_error_pct": null, "ttft_mape_pct": 36.88, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 29.82, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "091e65e5ecaa2710", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10843.64, "interactivity_tok_s": 100.54293183189222, "output_per_gpu": 10.258194444444445, "total_per_gpu": null, "tpot_error_pct": 63.4, "tpot_ms": 9.946, "tpot_relative": 1.634039, "ttft_error_pct": 36.88, "ttft_ms": 668.882, "ttft_relative": 1.368776, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8812.628871274932, "interactivity_tok_s": 108.51972051657293, "output_per_gpu": 11.268384594473922, "total_per_gpu": 101.29605719344299, "tpot_error_pct": 51.39, "tpot_ms": 9.214914996461697, "tpot_relative": 1.513928, "ttft_error_pct": 29.82, "ttft_ms": 342.95840625000426, "ttft_relative": 0.701818, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 6085.398158452517, "interactivity_tok_s": 164.29109208402838, "output_per_gpu": 18.32997607272289, "total_per_gpu": 146.55569693140188, "tpot_ms": 6.086757275242529, "tpot_relative": 1.0, "ttft_ms": 488.6716489265382, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f05785f3c701169c", "is_multinode": true, "parallelism": {"attention_dp_size": 24, "moe_ep_size": 24, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external decode prediction failed: AIC static_gen evidence prediction failed: PerfDataNotAvailableError: perf database error: MoE data missing for MoeKey { quant: \"nvfp4\", distribution: \"power_law_1.01\", topk: 8, num_experts: 384, hidden_size: 7168, inter_size: 2048, moe_tp_size: 1, moe_ep_size: 24 } at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 15360, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 147345.70946836536, "interactivity_tok_s": 38.43953543904489, "output_per_gpu": 3801.804985508763, "total_per_gpu": 14671.914128981856, "tpot_ms": 26.014882557197915, "tpot_relative": 1.0, "ttft_ms": 123399.43482749883, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 58.21, "tpot_shape_error_pct": 5.32, "ttft_mape_pct": 38.31, "ttft_shape_error_pct": 62.23}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.53, "tpot_shape_error_pct": 4.07, "ttft_mape_pct": 26.08, "ttft_shape_error_pct": 33.38}, "framework": "dynamo-vllm", "id": "f9c8b16a210f9d40", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13632.338, "interactivity_tok_s": 78.91414141414141, "output_per_gpu": 64.68950000000001, "total_per_gpu": null, "tpot_error_pct": 54.11, "tpot_ms": 12.672, "tpot_relative": 1.541094, "ttft_error_pct": 10.44, "ttft_ms": 668.882, "ttft_relative": 0.895615, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11279.99522586877, "interactivity_tok_s": 87.06283549572169, "output_per_gpu": 70.24520820991138, "total_per_gpu": 628.1550183780416, "tpot_error_pct": 39.69, "tpot_ms": 11.485957174564348, "tpot_relative": 1.396855, "ttft_error_pct": 11.27, "ttft_ms": 662.7016069625004, "ttft_relative": 0.88734, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 8348.7530935623, "interactivity_tok_s": 121.61414687785448, "output_per_gpu": 106.19982537529653, "total_per_gpu": 844.6634926102521, "tpot_ms": 8.222727582872158, "tpot_relative": 1.0, "ttft_ms": 746.8407181401744, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20616.359, "interactivity_tok_s": 51.284681265705935, "output_per_gpu": 167.89958333333334, "total_per_gpu": null, "tpot_error_pct": 62.3, "tpot_ms": 19.499, "tpot_relative": 2.371354, "ttft_error_pct": 66.17, "ttft_ms": 668.882, "ttft_relative": 0.895615, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17218.54875211168, "interactivity_tok_s": 57.26019735158069, "output_per_gpu": 183.2324248371821, "total_per_gpu": 1654.2373281185526, "tpot_error_pct": 45.37, "tpot_ms": 17.46413820161929, "tpot_relative": 2.123886, "ttft_error_pct": 40.89, "ttft_ms": 1168.90360171172, "ttft_relative": 1.565131, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27848011161", "measured": {"e2e_ms": 13019.683593692242, "interactivity_tok_s": 83.23737756546188, "output_per_gpu": 272.9273718253662, "total_per_gpu": 2190.7784241235713, "tpot_ms": 12.013833559491369, "tpot_relative": 1.461052, "ttft_ms": 1977.449965242272, "ttft_relative": 2.647753, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 130}]}, {"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.35, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.48, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 85.71, "points": 6, "status_counts": {"failed": 1, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.08, "ttft_shape_error_pct": null}, "frameworks": ["dynamo-vllm"], "gpu_skus": ["b200"], "hf_model_paths": ["nvidia/Kimi-K2.6-NVFP4"], "model": "nvidia/Kimi-K2.6-NVFP4", "precisions": ["fp4"], "rows": 7, "workloads": [{"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.35, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.48, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 85.71, "points": 6, "status_counts": {"failed": 1, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.08, "ttft_shape_error_pct": null}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.35, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.48, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 85.71, "points": 6, "status_counts": {"failed": 1, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.32, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.08, "ttft_shape_error_pct": null}, "gpu": "b200", "precisions": ["fp4"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.09, "tpot_shape_error_pct": null, "ttft_mape_pct": 13.72, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ed3f0e63fd7e9122", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 125809.3797365443, "interactivity_tok_s": 41.8554308582659, "output_per_gpu": 893.2510496699854, "total_per_gpu": 8031.963240629883, "tpot_error_pct": 17.09, "tpot_ms": 23.89176217982984, "tpot_relative": 0.82915, "ttft_error_pct": 13.72, "ttft_ms": 103794.01630705522, "ttft_relative": 1.137188, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 117824.23603219203, "interactivity_tok_s": 34.704422287484014, "output_per_gpu": 1908.3730848676335, "total_per_gpu": 8581.122799654579, "tpot_ms": 28.814771550329056, "tpot_relative": 1.0, "ttft_ms": 91272.49549758126, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.52, "tpot_shape_error_pct": null, "ttft_mape_pct": 27.92, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e8792e93e242aa26", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 96070.90345675978, "interactivity_tok_s": 46.91172676805205, "output_per_gpu": 779.6797285193389, "total_per_gpu": 7015.6784522608095, "tpot_error_pct": 11.52, "tpot_ms": 21.316631658952762, "tpot_relative": 0.88477, "ttft_error_pct": 27.92, "ttft_ms": 76442.1481181022, "ttft_relative": 0.720753, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30495901633", "measured": {"e2e_ms": 128237.33247728607, "interactivity_tok_s": 41.506083903728914, "output_per_gpu": 747.8174561775431, "total_per_gpu": 4486.433412640935, "tpot_ms": 24.092853527676695, "tpot_relative": 1.0, "ttft_ms": 106058.66706015797, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.35, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.48, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.57, "tpot_shape_error_pct": null, "ttft_mape_pct": 9.52, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "433457252d397aec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 8279.81, "interactivity_tok_s": 142.10601108426886, "output_per_gpu": 8.1843125, "total_per_gpu": null, "tpot_error_pct": 13.35, "tpot_ms": 7.037, "tpot_relative": 1.133471, "ttft_error_pct": 97.48, "ttft_ms": 1080.959, "ttft_relative": 1.974847, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6528.6515887, "interactivity_tok_s": 154.02758984350785, "output_per_gpu": 8.905935507515377, "total_per_gpu": 79.7216688513222, "tpot_error_pct": 4.57, "tpot_ms": 6.492343358848896, "tpot_relative": 1.045742, "ttft_error_pct": 9.52, "ttft_ms": 495.2468948000002, "ttft_relative": 0.904786, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 6316.821078211069, "interactivity_tok_s": 161.07311038605442, "output_per_gpu": 18.40646447207614, "total_per_gpu": 82.40271227916017, "tpot_ms": 6.208360896509881, "tpot_relative": 1.0, "ttft_ms": 547.3635403439403, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.53, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.73, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "071501dae288fae2", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 54518.50514738452, "interactivity_tok_s": 37.39877829210444, "output_per_gpu": 412.9788060982117, "total_per_gpu": 3716.8392574150776, "tpot_error_pct": 51.53, "tpot_ms": 26.73884136506989, "tpot_relative": 1.515267, "ttft_error_pct": 16.73, "ttft_ms": 29914.460520656772, "ttft_relative": 0.832678, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 52178.72518854201, "interactivity_tok_s": 56.669120369336994, "output_per_gpu": 541.1425576105585, "total_per_gpu": 3896.72610195775, "tpot_ms": 17.646294727755972, "tpot_relative": 1.0, "ttft_ms": 35925.60045463415, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 1, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.53, "tpot_shape_error_pct": null, "ttft_mape_pct": 55.57, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "31141066609b126d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (179790544896 bytes) meets/exceeds the KV-cache memory limit (capacity=193273528320 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 8256.230677626445, "interactivity_tok_s": 125.57611860436316, "output_per_gpu": 107.19604406920303, "total_per_gpu": 852.2780634205265, "tpot_ms": 7.963297568947595, "tpot_relative": 1.0, "ttft_ms": 894.8142231587553, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14055.490290593798, "interactivity_tok_s": 72.63396995125017, "output_per_gpu": 224.77809469970828, "total_per_gpu": 2029.3150359497408, "tpot_error_pct": 19.53, "tpot_ms": 13.767662715822519, "tpot_relative": 1.72889, "ttft_error_pct": 55.57, "ttft_ms": 1403.0348409757848, "ttft_relative": 1.567962, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 13742.854554550286, "interactivity_tok_s": 86.81627823864027, "output_per_gpu": 257.7396789899326, "total_per_gpu": 2068.982370200968, "tpot_ms": 11.518577164194987, "tpot_relative": 1.446458, "ttft_ms": 3157.748037739657, "ttft_relative": 3.528943, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 9.04, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "8b09e93ee7cd6d5f", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 251030.80779318503, "interactivity_tok_s": 23.220184925717177, "output_per_gpu": 1195.5248668537859, "total_per_gpu": 10760.99529106169, "tpot_error_pct": 5.68, "tpot_ms": 43.06597915559512, "tpot_relative": 0.943234, "ttft_error_pct": 9.04, "ttft_ms": 211385.16621170525, "ttft_relative": 1.090439, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30669586535", "measured": {"e2e_ms": 235884.99197841238, "interactivity_tok_s": 21.90206841844876, "output_per_gpu": 3811.2073673872296, "total_per_gpu": 11436.247774116375, "tpot_ms": 45.65778815473293, "tpot_relative": 1.0, "ttft_ms": 193853.321508713, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 7}]}, {"aic": {"coverage_pct": 70.77, "points": 184, "status_counts": {"failed": 76, "success": 184, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.52, "tpot_shape_error_pct": 7.57, "ttft_mape_pct": 54.75, "ttft_shape_error_pct": 35.59}, "aisimulate": {"coverage_pct": 100.0, "points": 260, "status_counts": {"failed": 0, "success": 260, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.88, "tpot_shape_error_pct": 8.0, "ttft_mape_pct": 36.8, "ttft_shape_error_pct": 19.46}, "frameworks": ["dynamo-vllm", "trt", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300"], "hf_model_paths": ["nvidia/MiniMax-M2.5-NVFP4"], "model": "nvidia/MiniMax-M2.5-NVFP4", "precisions": ["fp4"], "rows": 260, "workloads": [{"aic": {"coverage_pct": 66.88, "points": 107, "status_counts": {"failed": 53, "success": 107, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.46, "tpot_shape_error_pct": 7.85, "ttft_mape_pct": 61.45, "ttft_shape_error_pct": 33.47}, "aisimulate": {"coverage_pct": 100.0, "points": 160, "status_counts": {"failed": 0, "success": 160, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.68, "tpot_shape_error_pct": 7.99, "ttft_mape_pct": 38.48, "ttft_shape_error_pct": 17.11}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 73.21, "points": 41, "status_counts": {"failed": 15, "success": 41, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.21, "tpot_shape_error_pct": 9.26, "ttft_mape_pct": 67.86, "ttft_shape_error_pct": 32.32}, "aisimulate": {"coverage_pct": 100.0, "points": 56, "status_counts": {"failed": 0, "success": 56, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.81, "tpot_shape_error_pct": 9.2, "ttft_mape_pct": 44.64, "ttft_shape_error_pct": 21.4}, "gpu": "b200", "precisions": ["fp4"], "rows": 56, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.35, "tpot_shape_error_pct": 11.12, "ttft_mape_pct": 35.81, "ttft_shape_error_pct": 2.44}, "framework": "dynamo-vllm", "id": "f7b36fcfcba40ae6", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10553.63838804334, "interactivity_tok_s": 88.9558461538049, "output_per_gpu": 1542.911767885793, "total_per_gpu": 3084.332316552433, "tpot_error_pct": 12.62, "tpot_ms": 11.241532099767758, "tpot_relative": 1.126156, "ttft_error_pct": 35.02, "ttft_ms": 201.7105858078126, "ttft_relative": 0.649768, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 9503.591351071009, "interactivity_tok_s": 100.1781990967519, "output_per_gpu": 1987.27890052969, "total_per_gpu": 3411.8596396941443, "tpot_ms": 9.982211788756574, "tpot_relative": 1.0, "ttft_ms": 310.43481569322466, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14585.963736232565, "interactivity_tok_s": 66.02896079535203, "output_per_gpu": 4449.552545720986, "total_per_gpu": 8898.843037660843, "tpot_error_pct": 0.09, "tpot_ms": 15.144869583808335, "tpot_relative": 1.517186, "ttft_error_pct": 36.61, "ttft_ms": 639.0071061682603, "ttft_relative": 2.058426, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 14939.762060532394, "interactivity_tok_s": 66.08924720948987, "output_per_gpu": 5079.908166286833, "total_per_gpu": 8725.492503115196, "tpot_ms": 15.131054478956273, "tpot_relative": 1.515802, "ttft_ms": 1007.9837318270619, "ttft_relative": 3.247006, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.35, "tpot_shape_error_pct": 3.47, "ttft_mape_pct": 38.76, "ttft_shape_error_pct": 29.77}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.36, "tpot_shape_error_pct": 3.21, "ttft_mape_pct": 43.38, "ttft_shape_error_pct": 11.65}, "framework": "dynamo-vllm", "id": "9198fb269d66e62e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6502.303, "interactivity_tok_s": 159.15963711602737, "output_per_gpu": 65.10822222222222, "total_per_gpu": null, "tpot_error_pct": 13.25, "tpot_ms": 6.283, "tpot_relative": 1.132469, "ttft_error_pct": 19.61, "ttft_ms": 74.794, "ttft_relative": 0.803852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5386.825942224989, "interactivity_tok_s": 171.71685336740606, "output_per_gpu": 73.24923531968828, "total_per_gpu": 147.2148201852582, "tpot_error_pct": 4.97, "tpot_ms": 5.823540208137847, "tpot_relative": 1.049654, "ttft_error_pct": 48.21, "ttft_ms": 48.186658475002325, "ttft_relative": 0.517889, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 5179.0925577050075, "interactivity_tok_s": 180.24329048710152, "output_per_gpu": 85.03685806049337, "total_per_gpu": 152.4924156627883, "tpot_ms": 5.548056725426689, "tpot_relative": 1.0, "ttft_ms": 93.0444608675316, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6906.388, "interactivity_tok_s": 149.7454327643007, "output_per_gpu": 122.67977777777777, "total_per_gpu": null, "tpot_error_pct": 17.31, "tpot_ms": 6.678, "tpot_relative": 1.203665, "ttft_error_pct": 24.19, "ttft_ms": 74.794, "ttft_relative": 0.803852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5786.634671086686, "interactivity_tok_s": 161.90063323911107, "output_per_gpu": 139.30076458461176, "total_per_gpu": 277.572600937421, "tpot_error_pct": 8.51, "tpot_ms": 6.1766280958462945, "tpot_relative": 1.113296, "ttft_error_pct": 45.43, "ttft_ms": 53.84160494999992, "ttft_relative": 0.578665, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 5367.48036891222, "interactivity_tok_s": 175.6708222261606, "output_per_gpu": 165.53881845557575, "total_per_gpu": 294.4758870668267, "tpot_ms": 5.692464959904319, "tpot_relative": 1.026029, "ttft_ms": 98.66520585201215, "ttft_relative": 1.060409, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7679.776, "interactivity_tok_s": 134.51708366962603, "output_per_gpu": 220.23733333333334, "total_per_gpu": null, "tpot_error_pct": 19.51, "tpot_ms": 7.434, "tpot_relative": 1.339929, "ttft_error_pct": 36.99, "ttft_ms": 74.794, "ttft_relative": 0.803852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6320.278886156217, "interactivity_tok_s": 146.2791943890952, "output_per_gpu": 251.15513156171858, "total_per_gpu": 504.98389478947564, "tpot_error_pct": 9.9, "tpot_ms": 6.836242188619463, "tpot_relative": 1.232187, "ttft_error_pct": 44.86, "ttft_ms": 65.45815806875075, "ttft_relative": 0.703515, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 5805.075760772161, "interactivity_tok_s": 160.76576018736705, "output_per_gpu": 307.02968506579856, "total_per_gpu": 549.6721181220041, "tpot_ms": 6.220229972069512, "tpot_relative": 1.121155, "ttft_ms": 118.70450697315391, "ttft_relative": 1.275783, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8408.152, "interactivity_tok_s": 122.75963663147556, "output_per_gpu": 401.9541111111111, "total_per_gpu": null, "tpot_error_pct": 18.55, "tpot_ms": 8.146, "tpot_relative": 1.468262, "ttft_error_pct": 48.58, "ttft_ms": 74.794, "ttft_relative": 0.803852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7010.841560171883, "interactivity_tok_s": 133.2393292521499, "output_per_gpu": 454.14618399239066, "total_per_gpu": 906.8557848416399, "tpot_error_pct": 9.22, "tpot_ms": 7.505291460207981, "tpot_relative": 1.352778, "ttft_error_pct": 40.99, "ttft_ms": 85.8342447656255, "ttft_relative": 0.922508, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 6492.94705294742, "interactivity_tok_s": 145.52651763444626, "output_per_gpu": 555.1935108382143, "total_per_gpu": 986.527020677893, "tpot_ms": 6.871599872347244, "tpot_relative": 1.23856, "ttft_ms": 145.46898001935915, "ttft_relative": 1.563435, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9430.129, "interactivity_tok_s": 109.34937124111536, "output_per_gpu": 715.9807777777778, "total_per_gpu": null, "tpot_error_pct": 13.15, "tpot_ms": 9.145, "tpot_relative": 1.648325, "ttft_error_pct": 64.41, "ttft_ms": 74.794, "ttft_relative": 0.803852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7887.000867800013, "interactivity_tok_s": 118.725402594461, "output_per_gpu": 800.3365103893113, "total_per_gpu": 1601.0338743126078, "tpot_error_pct": 4.21, "tpot_ms": 8.422797296512632, "tpot_relative": 1.518153, "ttft_error_pct": 37.42, "ttft_ms": 131.49650600625023, "ttft_relative": 1.413265, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 7649.438703989654, "interactivity_tok_s": 123.72784527349023, "output_per_gpu": 936.158018977544, "total_per_gpu": 1668.4490751178591, "tpot_ms": 8.08225503151358, "tpot_relative": 1.456772, "ttft_ms": 210.13366855513596, "ttft_relative": 2.258422, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.85, "tpot_shape_error_pct": 4.69, "ttft_mape_pct": 68.82, "ttft_shape_error_pct": 51.51}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.93, "tpot_shape_error_pct": 4.7, "ttft_mape_pct": 35.48, "ttft_shape_error_pct": 11.51}, "framework": "dynamo-vllm", "id": "51a703cce55dd428", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9547.774, "interactivity_tok_s": 107.9913606911447, "output_per_gpu": 636.4692, "total_per_gpu": null, "tpot_error_pct": 14.5, "tpot_ms": 9.26, "tpot_relative": 1.144974, "ttft_error_pct": 49.2, "ttft_ms": 74.794, "ttft_relative": 0.508042, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7953.690085118731, "interactivity_tok_s": 117.29230775352431, "output_per_gpu": 721.7363300608195, "total_per_gpu": 1441.1896193693347, "tpot_error_pct": 5.42, "tpot_ms": 8.525708285162057, "tpot_relative": 1.054181, "ttft_error_pct": 40.61, "ttft_ms": 87.43869755937484, "ttft_relative": 0.593932, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 7617.270329100211, "interactivity_tok_s": 123.64731663222284, "output_per_gpu": 948.7279718229287, "total_per_gpu": 1517.220906016845, "tpot_ms": 8.087518817528444, "tpot_relative": 1.0, "ttft_ms": 147.22013347927714, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11629.579, "interactivity_tok_s": 88.53474988933156, "output_per_gpu": 1043.6362, "total_per_gpu": null, "tpot_error_pct": 21.76, "tpot_ms": 11.295, "tpot_relative": 1.396596, "ttft_error_pct": 64.77, "ttft_ms": 74.794, "ttft_relative": 0.508042, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9689.690695290652, "interactivity_tok_s": 96.31841440332522, "output_per_gpu": 1177.7526297693337, "total_per_gpu": 2356.036281418959, "tpot_error_pct": 11.92, "tpot_ms": 10.382230710449452, "tpot_relative": 1.283735, "ttft_error_pct": 38.64, "ttft_ms": 130.24897093124991, "ttft_relative": 0.884723, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 8750.726274555018, "interactivity_tok_s": 107.79707345804438, "output_per_gpu": 1636.8575778825773, "total_per_gpu": 2625.531278801148, "tpot_ms": 9.276689690367236, "tpot_relative": 1.147038, "ttft_ms": 212.27316609765694, "ttft_relative": 1.441876, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13342.081, "interactivity_tok_s": 77.10694733595497, "output_per_gpu": 1817.8376, "total_per_gpu": null, "tpot_error_pct": 20.01, "tpot_ms": 12.969, "tpot_relative": 1.603582, "ttft_error_pct": 76.26, "ttft_ms": 74.794, "ttft_relative": 0.508042, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11065.826016335119, "interactivity_tok_s": 84.60709486790067, "output_per_gpu": 2054.1862269100593, "total_per_gpu": 4112.885170579884, "tpot_error_pct": 9.37, "tpot_ms": 11.819339755860035, "tpot_relative": 1.46143, "ttft_error_pct": 34.91, "ttft_ms": 205.05567917187477, "ttft_relative": 1.392851, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 10245.826310571465, "interactivity_tok_s": 92.53510991808692, "output_per_gpu": 2789.79798945003, "total_per_gpu": 4478.8971581312935, "tpot_ms": 10.806708944153314, "tpot_relative": 1.336221, "ttft_ms": 315.05640314762786, "ttft_relative": 2.140036, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15286.804, "interactivity_tok_s": 67.24949562878278, "output_per_gpu": 3170.8527999999997, "total_per_gpu": null, "tpot_error_pct": 11.15, "tpot_ms": 14.87, "tpot_relative": 1.838636, "ttft_error_pct": 85.07, "ttft_ms": 74.794, "ttft_relative": 0.508042, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12805.74895470663, "interactivity_tok_s": 74.0025466758512, "output_per_gpu": 3551.1088645129007, "total_per_gpu": 7098.785593826442, "tpot_error_pct": 1.0, "tpot_ms": 13.51304846818635, "tpot_relative": 1.670852, "ttft_error_pct": 27.75, "ttft_ms": 361.93600207617095, "ttft_relative": 2.458468, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 12822.192450525654, "interactivity_tok_s": 74.74534609813283, "output_per_gpu": 4438.799243665021, "total_per_gpu": 7112.702022777819, "tpot_ms": 13.378759377033381, "tpot_relative": 1.654248, "ttft_ms": 500.9469599828662, "ttft_relative": 3.402707, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 51.76, "tpot_shape_error_pct": 66.04, "ttft_mape_pct": 96.75, "ttft_shape_error_pct": 90.76}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.55, "tpot_shape_error_pct": 46.67, "ttft_mape_pct": 55.24, "ttft_shape_error_pct": 53.09}, "framework": "dynamo-vllm", "id": "5f990e43a78c90c6", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 19514.863, "interactivity_tok_s": 52.62327000999842, "output_per_gpu": 4435.8451000000005, "total_per_gpu": null, "tpot_error_pct": 5.37, "tpot_ms": 19.003, "tpot_relative": 1.053676, "ttft_error_pct": 91.77, "ttft_ms": 74.794, "ttft_relative": 0.082266, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16579.464474673438, "interactivity_tok_s": 57.73834709653693, "output_per_gpu": 5480.291889926018, "total_per_gpu": 10960.261021283239, "tpot_error_pct": 3.97, "tpot_ms": 17.31951207969337, "tpot_relative": 0.96033, "ttft_error_pct": 30.72, "ttft_ms": 629.8877490549836, "ttft_relative": 0.692813, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 17514.669696563535, "interactivity_tok_s": 55.447871772782065, "output_per_gpu": 6467.890751535722, "total_per_gpu": 10368.920405982139, "tpot_ms": 18.034957303643065, "tpot_relative": 1.0, "ttft_ms": 909.1747386431507, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30428.227, "interactivity_tok_s": 33.7029422668599, "output_per_gpu": 4435.8451000000005, "total_per_gpu": null, "tpot_error_pct": 36.18, "tpot_ms": 29.671, "tpot_relative": 1.645194, "ttft_error_pct": 98.73, "ttft_ms": 74.794, "ttft_relative": 0.082266, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25795.464390933303, "interactivity_tok_s": 37.882051103742434, "output_per_gpu": 7097.790859000304, "total_per_gpu": 14192.469642993561, "tpot_error_pct": 21.16, "tpot_ms": 26.39772585865099, "tpot_relative": 1.463698, "ttft_error_pct": 75.0, "ttft_ms": 1473.5728052342845, "ttft_relative": 1.620781, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 25969.589530710662, "interactivity_tok_s": 45.89672220007666, "output_per_gpu": 8737.984413280477, "total_per_gpu": 14002.183227965204, "tpot_ms": 21.788048297669715, "tpot_relative": 1.208101, "ttft_ms": 5893.536600073412, "ttft_relative": 6.482293, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48195.691, "interactivity_tok_s": 21.25895533493484, "output_per_gpu": 4435.8451000000005, "total_per_gpu": null, "tpot_error_pct": 113.73, "tpot_ms": 47.039, "tpot_relative": 2.608212, "ttft_error_pct": 99.75, "ttft_ms": 74.794, "ttft_relative": 0.082266, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44440.323924694734, "interactivity_tok_s": 28.30318456569094, "output_per_gpu": 8238.280655485629, "total_per_gpu": 16482.54800729677, "tpot_error_pct": 60.54, "tpot_ms": 35.33171320983427, "tpot_relative": 1.959068, "ttft_error_pct": 60.01, "ttft_ms": 11922.394503863412, "ttft_relative": 13.113425, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 50066.80957827996, "interactivity_tok_s": 45.43735741870509, "output_per_gpu": 9010.514654790988, "total_per_gpu": 14448.920720766337, "tpot_ms": 22.008322156260178, "tpot_relative": 1.220315, "ttft_ms": 29809.903693422166, "ttft_relative": 32.787871, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.51, "tpot_shape_error_pct": 1.98, "ttft_mape_pct": 40.63, "ttft_shape_error_pct": 37.42}, "framework": "dynamo-vllm", "id": "0c391e99934e91d5", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 44461.67400038529, "interactivity_tok_s": 28.278736837168598, "output_per_gpu": 10256.492012484992, "total_per_gpu": 20520.437340244323, "tpot_error_pct": 17.68, "tpot_ms": 35.362258426113094, "tpot_relative": 1.176767, "ttft_error_pct": 49.99, "ttft_ms": 11914.926907751436, "ttft_relative": 0.50015, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 51481.25686306582, "interactivity_tok_s": 33.27748614038987, "output_per_gpu": 11722.012299314752, "total_per_gpu": 17622.164863377562, "tpot_ms": 30.050346825515476, "tpot_relative": 1.0, "ttft_ms": 23822.725781330668, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 65184.5721731021, "interactivity_tok_s": 27.874422666142372, "output_per_gpu": 10452.433303858614, "total_per_gpu": 20907.72498519214, "tpot_error_pct": 15.35, "tpot_ms": 35.875182491748916, "tpot_relative": 1.193836, "ttft_error_pct": 31.27, "ttft_ms": 32159.46955664184, "ttft_relative": 1.349949, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 75419.83377481002, "interactivity_tok_s": 32.15346693135894, "output_per_gpu": 11986.762139803604, "total_per_gpu": 18015.76146069493, "tpot_ms": 31.100845272293498, "tpot_relative": 1.034958, "ttft_ms": 46789.73784344805, "ttft_relative": 1.96408, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.19, "tpot_shape_error_pct": 10.46, "ttft_mape_pct": 37.69, "ttft_shape_error_pct": 23.6}, "framework": "dynamo-vllm", "id": "fad2eb35c5f6c11b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7469.689060709401, "interactivity_tok_s": 125.4784802794652, "output_per_gpu": 583.9046967843514, "total_per_gpu": 1168.072662919295, "tpot_error_pct": 1.35, "tpot_ms": 7.969494034138792, "tpot_relative": 1.013548, "ttft_error_pct": 37.88, "ttft_ms": 131.47638870156328, "ttft_relative": 0.621236, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 7448.868219515134, "interactivity_tok_s": 127.17843891036513, "output_per_gpu": 639.9842153926771, "total_per_gpu": 1184.468383075235, "tpot_ms": 7.862968035838183, "tpot_relative": 1.0, "ttft_ms": 211.63675285424688, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8672.838235310219, "interactivity_tok_s": 108.5417330559608, "output_per_gpu": 1006.1482383220862, "total_per_gpu": 2014.506822502013, "tpot_error_pct": 5.63, "tpot_ms": 9.213046188275163, "tpot_relative": 1.171701, "ttft_error_pct": 33.17, "ttft_ms": 206.9937129968761, "ttft_relative": 0.978061, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 8324.701732703761, "interactivity_tok_s": 114.65505995214187, "output_per_gpu": 1141.916032795278, "total_per_gpu": 2115.3411162823145, "tpot_ms": 8.721813066230219, "tpot_relative": 1.109227, "ttft_ms": 309.73717390916136, "ttft_relative": 1.463532, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10675.5054720645, "interactivity_tok_s": 89.30810111614763, "output_per_gpu": 1639.0154001039195, "total_per_gpu": 3276.446697139864, "tpot_error_pct": 12.1, "tpot_ms": 11.197192499921957, "tpot_relative": 1.424041, "ttft_error_pct": 27.07, "ttft_ms": 364.3063274156269, "ttft_relative": 1.721376, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 9698.885512386369, "interactivity_tok_s": 100.11219434095787, "output_per_gpu": 1945.3318398817585, "total_per_gpu": 3596.7538186355914, "tpot_ms": 9.988793139367642, "tpot_relative": 1.270359, "ttft_ms": 499.5008770663844, "ttft_relative": 2.36018, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12246.423949462182, "interactivity_tok_s": 79.91202944760204, "output_per_gpu": 2848.4724642236906, "total_per_gpu": 5694.320975289151, "tpot_error_pct": 12.92, "tpot_ms": 12.513760530330362, "tpot_relative": 1.591481, "ttft_error_pct": 73.97, "ttft_ms": 718.517908278515, "ttft_relative": 3.395053, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 12967.701839818801, "interactivity_tok_s": 90.23273197934427, "output_per_gpu": 2916.046282766638, "total_per_gpu": 5393.162842610186, "tpot_ms": 11.08245287562518, "tpot_relative": 1.409449, "ttft_ms": 2760.7603791565452, "ttft_relative": 13.044806, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 21805.442063831055, "interactivity_tok_s": 78.74963222100727, "output_per_gpu": 3185.7495686044913, "total_per_gpu": 6371.311514361156, "tpot_error_pct": 14.7, "tpot_ms": 12.69847200293641, "tpot_relative": 1.614972, "ttft_error_pct": 32.58, "ttft_ms": 10111.093857917314, "ttft_relative": 47.775699, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 25190.829942094704, "interactivity_tok_s": 90.32663646224631, "output_per_gpu": 2983.372032820846, "total_per_gpu": 5518.56530229869, "tpot_ms": 11.070931445764268, "tpot_relative": 1.407984, "ttft_ms": 14997.419288918105, "ttft_relative": 70.863964, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42458.19631080012, "interactivity_tok_s": 78.53278279889619, "output_per_gpu": 3263.745860280619, "total_per_gpu": 6526.060708839011, "tpot_error_pct": 14.44, "tpot_ms": 12.733535784167518, "tpot_relative": 1.619431, "ttft_error_pct": 21.49, "ttft_ms": 30726.43109145095, "ttft_relative": 145.18476, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 49387.68221307647, "interactivity_tok_s": 89.87534608249503, "output_per_gpu": 3037.8540515402024, "total_per_gpu": 5616.933763109498, "tpot_ms": 11.126521828156491, "tpot_relative": 1.415054, "ttft_ms": 39135.3437362305, "ttft_relative": 184.917521, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.53, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.89, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.02, "tpot_shape_error_pct": null, "ttft_mape_pct": 44.86, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "d84425953d6e4d69", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8408.152, "interactivity_tok_s": 122.75963663147556, "output_per_gpu": 361.7588, "total_per_gpu": null, "tpot_error_pct": 19.53, "tpot_ms": 8.146, "tpot_relative": 1.195274, "ttft_error_pct": 35.89, "ttft_ms": 74.794, "ttft_relative": 0.641077, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6924.876747956236, "interactivity_tok_s": 133.3650578243935, "output_per_gpu": 413.07237317026886, "total_per_gpu": 830.5420420294865, "tpot_error_pct": 10.02, "tpot_ms": 7.498215921870146, "tpot_relative": 1.100224, "ttft_error_pct": 44.86, "ttft_ms": 64.33298430624998, "ttft_relative": 0.551413, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 6346.862392842013, "interactivity_tok_s": 146.73141074226993, "output_per_gpu": 563.4867718650828, "total_per_gpu": 907.9241657776167, "tpot_ms": 6.815173349327876, "tpot_relative": 1.0, "ttft_ms": 116.66929661878385, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.31, "tpot_shape_error_pct": 1.96, "ttft_mape_pct": 80.39, "ttft_shape_error_pct": 35.99}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.05, "tpot_shape_error_pct": 3.06, "ttft_mape_pct": 29.6, "ttft_shape_error_pct": 15.11}, "framework": "dynamo-vllm", "id": "789ed631d560b444", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11629.579, "interactivity_tok_s": 88.53474988933156, "output_per_gpu": 1159.5957777777776, "total_per_gpu": null, "tpot_error_pct": 22.52, "tpot_ms": 11.295, "tpot_relative": 1.22518, "ttft_error_pct": 76.09, "ttft_ms": 74.794, "ttft_relative": 0.239137, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9758.251623214877, "interactivity_tok_s": 96.18355702207336, "output_per_gpu": 1292.6015181310495, "total_per_gpu": 2588.0426739045647, "tpot_error_pct": 12.77, "tpot_ms": 10.396787465143424, "tpot_relative": 1.12775, "ttft_error_pct": 34.55, "ttft_ms": 204.71800088593733, "ttft_relative": 0.65454, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 8784.711426504145, "interactivity_tok_s": 108.4709675759501, "output_per_gpu": 1623.7298698821319, "total_per_gpu": 2896.473797780308, "tpot_ms": 9.219056696436414, "tpot_relative": 1.0, "ttft_ms": 312.76609181695676, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13342.081, "interactivity_tok_s": 77.10694733595497, "output_per_gpu": 2019.8195555555556, "total_per_gpu": null, "tpot_error_pct": 20.11, "tpot_ms": 12.969, "tpot_relative": 1.40676, "ttft_error_pct": 84.69, "ttft_ms": 74.794, "ttft_relative": 0.239137, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11237.855351264127, "interactivity_tok_s": 84.71871488079577, "output_per_gpu": 2248.6683750670627, "total_per_gpu": 4495.163419443287, "tpot_error_pct": 9.32, "tpot_ms": 11.803767342399597, "tpot_relative": 1.280366, "ttft_error_pct": 24.66, "ttft_ms": 368.14983161015573, "ttft_relative": 1.177077, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 10432.779638616375, "interactivity_tok_s": 92.61395655227179, "output_per_gpu": 2717.08949118313, "total_per_gpu": 4837.6065635788855, "tpot_ms": 10.797508682566594, "tpot_relative": 1.171216, "ttft_ms": 488.6246853693592, "ttft_relative": 1.562269, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.16, "tpot_shape_error_pct": 0.1, "ttft_mape_pct": 23.02, "ttft_shape_error_pct": 17.32}, "framework": "dynamo-vllm", "id": "375df69998acd31f", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 43652.6259612218, "interactivity_tok_s": 38.09077216917659, "output_per_gpu": 8312.240424495501, "total_per_gpu": 16620.83629025554, "tpot_error_pct": 8.09, "tpot_ms": 26.253077662972906, "tpot_relative": 1.080914, "ttft_error_pct": 30.99, "ttft_ms": 19464.26288714572, "ttft_relative": 0.690141, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 50582.83611670146, "interactivity_tok_s": 41.17283503512091, "output_per_gpu": 8948.690583342703, "total_per_gpu": 14339.82933271105, "tpot_ms": 24.287858709437625, "tpot_relative": 1.0, "ttft_ms": 28203.312557988775, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 85018.22116659011, "interactivity_tok_s": 37.85708185436647, "output_per_gpu": 8486.84507083753, "total_per_gpu": 16979.85746788371, "tpot_error_pct": 8.16, "tpot_ms": 26.415136904818223, "tpot_relative": 1.087586, "ttft_error_pct": 20.68, "ttft_ms": 60705.28297877892, "ttft_relative": 2.152417, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 99010.80335749596, "interactivity_tok_s": 40.94578643633672, "output_per_gpu": 9096.778603788354, "total_per_gpu": 14587.25032877163, "tpot_ms": 24.42253738500832, "tpot_relative": 1.005545, "ttft_ms": 76531.79606906687, "ttft_relative": 2.713575, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 167592.96947155503, "interactivity_tok_s": 37.703154221202546, "output_per_gpu": 8588.89897842175, "total_per_gpu": 17179.59807871652, "tpot_error_pct": 8.24, "tpot_ms": 26.522979858211578, "tpot_relative": 1.092026, "ttft_error_pct": 17.38, "ttft_ms": 143176.25766086482, "ttft_relative": 5.076576, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 195858.35122429256, "interactivity_tok_s": 40.808631090146775, "output_per_gpu": 9176.102406595332, "total_per_gpu": 14711.449551903384, "tpot_ms": 24.504620059197464, "tpot_relative": 1.008925, "ttft_ms": 173301.75931042334, "ttft_relative": 6.144731, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.09, "tpot_shape_error_pct": 4.7, "ttft_mape_pct": 91.5, "ttft_shape_error_pct": 45.92}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.75, "tpot_shape_error_pct": 3.61, "ttft_mape_pct": 35.08, "ttft_shape_error_pct": 0.81}, "framework": "dynamo-vllm", "id": "3e3f290dfe446adc", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 18834.568, "interactivity_tok_s": 54.53157378121932, "output_per_gpu": 4674.9435454545455, "total_per_gpu": null, "tpot_error_pct": 6.59, "tpot_ms": 18.338, "tpot_relative": 1.065881, "ttft_error_pct": 88.97, "ttft_ms": 74.794, "ttft_relative": 0.110293, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15806.807267326547, "interactivity_tok_s": 59.92206794471977, "output_per_gpu": 5223.673633625957, "total_per_gpu": 10447.039622064445, "tpot_error_pct": 3.0, "tpot_ms": 16.688342613985476, "tpot_relative": 0.969996, "ttft_error_pct": 35.35, "ttft_ms": 438.4358602876976, "ttft_relative": 0.64653, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 16518.87363644928, "interactivity_tok_s": 58.124144081667474, "output_per_gpu": 6869.290343205608, "total_per_gpu": 10011.290254704983, "tpot_ms": 17.204554420533874, "tpot_relative": 1.0, "ttft_ms": 678.1372304035813, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26630.851, "interactivity_tok_s": 38.52228514195462, "output_per_gpu": 6048.879727272727, "total_per_gpu": null, "tpot_error_pct": 11.59, "tpot_ms": 25.959, "tpot_relative": 1.508845, "ttft_error_pct": 94.04, "ttft_ms": 74.794, "ttft_relative": 0.110293, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22356.077332368026, "interactivity_tok_s": 42.775670244621345, "output_per_gpu": 7422.729079890984, "total_per_gpu": 14842.203613386826, "tpot_error_pct": 0.5, "tpot_ms": 23.377775129677623, "tpot_relative": 1.358813, "ttft_error_pct": 34.82, "ttft_ms": 817.297497794191, "ttft_relative": 1.20521, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 22688.35488959105, "interactivity_tok_s": 42.98825953931397, "output_per_gpu": 10017.102460956356, "total_per_gpu": 14592.641780745336, "tpot_ms": 23.262165314822106, "tpot_relative": 1.352093, "ttft_ms": 1253.913238126961, "ttft_relative": 1.849055, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 11}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.43, "tpot_shape_error_pct": 1.78, "ttft_mape_pct": 55.41, "ttft_shape_error_pct": 0.0}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.18, "tpot_shape_error_pct": 1.74, "ttft_mape_pct": 11.52, "ttft_shape_error_pct": 2.81}, "framework": "trt", "id": "047872a4ee3c913c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8062.809299420257, "interactivity_tok_s": 128.34992362734812, "output_per_gpu": 505.80641819032627, "total_per_gpu": null, "tpot_error_pct": 5.37, "tpot_ms": 7.79120058460966, "tpot_relative": 1.053711, "ttft_error_pct": 55.41, "ttft_ms": 92.41110136457459, "ttft_relative": 1.554131, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7243.935115624974, "interactivity_tok_s": 127.47287420685511, "output_per_gpu": 492.5502122812091, "total_per_gpu": 989.9173775224267, "tpot_error_pct": 6.1, "tpot_ms": 7.844806247776776, "tpot_relative": 1.060961, "ttft_error_pct": 12.74, "ttft_ms": 51.884569675000265, "ttft_relative": 0.872573, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 6838.6649203748675, "interactivity_tok_s": 135.24369749961014, "output_per_gpu": 521.8970258583878, "total_per_gpu": 1048.898005305355, "tpot_ms": 7.394059896971411, "tpot_relative": 1.0, "ttft_ms": 59.46157420007694, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9633.747517221067, "interactivity_tok_s": 107.32125815091275, "output_per_gpu": 848.1721170471317, "total_per_gpu": null, "tpot_error_pct": 3.5, "tpot_ms": 9.317818456748078, "tpot_relative": 1.260176, "ttft_error_pct": 55.41, "ttft_ms": 101.61923596778313, "ttft_relative": 1.70899, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8772.710338160625, "interactivity_tok_s": 106.53936220317694, "output_per_gpu": 830.8928597888911, "total_per_gpu": 1655.648430069076, "tpot_error_pct": 4.25, "tpot_ms": 9.386202238501674, "tpot_relative": 1.269425, "ttft_error_pct": 10.29, "ttft_ms": 58.656757937499876, "ttft_relative": 0.986465, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 8422.869196175088, "interactivity_tok_s": 111.07246750576783, "output_per_gpu": 865.4873878736752, "total_per_gpu": 1724.5819579454687, "tpot_ms": 9.00313122104784, "tpot_relative": 1.217617, "ttft_ms": 65.38645565051411, "ttft_relative": 1.099642, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.11, "tpot_shape_error_pct": 8.2, "ttft_mape_pct": 63.14, "ttft_shape_error_pct": 10.81}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 67.18, "tpot_shape_error_pct": 17.91, "ttft_mape_pct": 68.83, "ttft_shape_error_pct": 26.16}, "framework": "trt", "id": "0e757c0ef538c55f", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 11400.783274329242, "interactivity_tok_s": 90.75283169490763, "output_per_gpu": 1436.3306292863674, "total_per_gpu": null, "tpot_error_pct": 3.73, "tpot_ms": 11.018939919823048, "tpot_relative": 1.037275, "ttft_error_pct": 64.47, "ttft_ms": 128.40773635026318, "ttft_relative": 0.355347, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14499.47073306556, "interactivity_tok_s": 63.871469708806494, "output_per_gpu": 985.8190529505629, "total_per_gpu": 1973.8037918083076, "tpot_error_pct": 47.38, "tpot_ms": 15.656442611373347, "tpot_relative": 1.473829, "ttft_error_pct": 73.94, "ttft_ms": 94.16093086015583, "ttft_relative": 0.260575, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 10126.482136128221, "interactivity_tok_s": 94.1356050680225, "output_per_gpu": 1393.51021857109, "total_per_gpu": 2790.081754970072, "tpot_ms": 10.622973095858882, "tpot_relative": 1.0, "ttft_ms": 361.35865468440613, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14666.885608450119, "interactivity_tok_s": 70.57433781595915, "output_per_gpu": 2239.7107658789787, "total_per_gpu": null, "tpot_error_pct": 9.75, "tpot_ms": 14.169456362562816, "tpot_relative": 1.33385, "ttft_error_pct": 61.35, "ttft_ms": 171.53174954835862, "ttft_relative": 0.474686, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19406.802001693784, "interactivity_tok_s": 47.84730307150564, "output_per_gpu": 1483.288431180256, "total_per_gpu": 2965.1432689028934, "tpot_error_pct": 61.89, "tpot_ms": 20.89981954689369, "tpot_relative": 1.967417, "ttft_error_pct": 73.07, "ttft_ms": 119.5310771066409, "ttft_relative": 0.330782, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 12338.688604903122, "interactivity_tok_s": 77.4586914370969, "output_per_gpu": 2297.470933787813, "total_per_gpu": 4592.721369370077, "tpot_ms": 12.910107070580786, "tpot_relative": 1.215301, "ttft_ms": 443.80060761248285, "ttft_relative": 1.228144, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18081.2693222098, "interactivity_tok_s": 57.23642025684001, "output_per_gpu": 3638.4818808011714, "total_per_gpu": null, "tpot_error_pct": 11.33, "tpot_ms": 17.471393135920227, "tpot_relative": 1.64468, "ttft_error_pct": 59.18, "ttft_ms": 208.03414416340956, "ttft_relative": 0.5757, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25922.895502289655, "interactivity_tok_s": 35.83627209099287, "output_per_gpu": 2229.8126252703614, "total_per_gpu": 4457.571193865145, "tpot_error_pct": 77.81, "tpot_ms": 27.904688229313372, "tpot_relative": 2.626825, "ttft_error_pct": 66.83, "ttft_ms": 169.0187237230462, "ttft_relative": 0.467731, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 14976.942355009589, "interactivity_tok_s": 63.72162324210909, "output_per_gpu": 3799.690978174066, "total_per_gpu": 7595.881760622984, "tpot_ms": 15.693259981788586, "tpot_relative": 1.477295, "ttft_ms": 509.6021156130121, "ttft_relative": 1.410239, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24191.581856626733, "interactivity_tok_s": 42.68363478759292, "output_per_gpu": 5435.9351056332225, "total_per_gpu": null, "tpot_error_pct": 15.63, "tpot_ms": 23.42818283813719, "tpot_relative": 2.205426, "ttft_error_pct": 67.58, "ttft_ms": 224.55081321238737, "ttft_relative": 0.621407, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34215.6969364453, "interactivity_tok_s": 27.172441718122613, "output_per_gpu": 3387.867517473839, "total_per_gpu": 6775.535508482085, "tpot_error_pct": 81.63, "tpot_ms": 36.80199263554043, "tpot_relative": 3.464378, "ttft_error_pct": 61.48, "ttft_ms": 266.8384185706053, "ttft_relative": 0.738431, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 19368.743515984308, "interactivity_tok_s": 49.3539907430589, "output_per_gpu": 5890.634186077591, "total_per_gpu": 11780.921446718108, "tpot_ms": 20.26178602670827, "tpot_relative": 1.907355, "ttft_ms": 692.714326071657, "ttft_relative": 1.916972, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.8, "tpot_shape_error_pct": null, "ttft_mape_pct": 416.28, "ttft_shape_error_pct": null}, "framework": "trt", "id": "36f1a43bd902cc65", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 59988.94475108478, "interactivity_tok_s": 18.069233422093017, "output_per_gpu": 7687.834830621946, "total_per_gpu": 15375.216890731495, "tpot_error_pct": 19.8, "tpot_ms": 55.34269089563661, "tpot_relative": 1.198019, "ttft_error_pct": 416.28, "ttft_ms": 8959.96407008905, "ttft_relative": 5.16277, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 44302.36064716611, "interactivity_tok_s": 21.64728915101195, "output_per_gpu": 10366.841097991772, "total_per_gpu": 20733.071646972116, "tpot_ms": 46.19516065147828, "tpot_relative": 1.0, "ttft_ms": 1735.4954449806842, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.17, "tpot_shape_error_pct": null, "ttft_mape_pct": 71.88, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.3, "ttft_shape_error_pct": null}, "framework": "trt", "id": "c9316b75b3a58f38", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 39359.268411752055, "interactivity_tok_s": 26.15308888070336, "output_per_gpu": 6677.079525815748, "total_per_gpu": null, "tpot_error_pct": 33.17, "tpot_ms": 38.236401235872144, "tpot_relative": 1.331686, "ttft_error_pct": 71.88, "ttft_ms": 243.42994745485282, "ttft_relative": 0.281215, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46162.15924999882, "interactivity_tok_s": 20.169099914196327, "output_per_gpu": 5023.11054343639, "total_per_gpu": 10045.925253730591, "tpot_error_pct": 72.68, "tpot_ms": 49.58079459441494, "tpot_relative": 1.726786, "ttft_error_pct": 49.3, "ttft_ms": 438.8454884424782, "ttft_relative": 0.506963, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 27327.192078451277, "interactivity_tok_s": 34.82771107553186, "output_per_gpu": 8403.920762675074, "total_per_gpu": 16807.346581377795, "tpot_ms": 28.71276834217647, "tpot_relative": 1.0, "ttft_ms": 865.6363492955294, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.43, "tpot_shape_error_pct": 0.05, "ttft_mape_pct": 22.36, "ttft_shape_error_pct": 9.5}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.76, "tpot_shape_error_pct": 0.22, "ttft_mape_pct": 31.05, "ttft_shape_error_pct": 8.29}, "framework": "trt", "id": "79003897f313d233", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6141.176179294055, "interactivity_tok_s": 168.33277847087967, "output_per_gpu": 166.10054147928966, "total_per_gpu": null, "tpot_error_pct": 5.41, "tpot_ms": 5.940613640931452, "tpot_relative": 0.94593, "ttft_error_pct": 28.46, "ttft_ms": 63.92842462118012, "ttft_relative": 1.284589, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5525.603818425041, "interactivity_tok_s": 166.99636754331786, "output_per_gpu": 160.72863903482488, "total_per_gpu": 323.0291427734963, "tpot_error_pct": 4.65, "tpot_ms": 5.988154201860744, "tpot_relative": 0.9535, "ttft_error_pct": 28.06, "ttft_ms": 35.7995839749999, "ttft_relative": 0.719363, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 5807.12944927518, "interactivity_tok_s": 159.231015396836, "output_per_gpu": 153.35643419115564, "total_per_gpu": 308.21263573839906, "tpot_ms": 6.28018352773671, "tpot_relative": 1.0, "ttft_ms": 49.765678624999055, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6731.16436470278, "interactivity_tok_s": 153.58404682334168, "output_per_gpu": 303.4896437257684, "total_per_gpu": null, "tpot_error_pct": 5.45, "tpot_ms": 6.511092920674494, "tpot_relative": 1.036768, "ttft_error_pct": 16.26, "ttft_ms": 70.31630685277345, "ttft_relative": 1.412948, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6121.933214899969, "interactivity_tok_s": 152.63031701458215, "output_per_gpu": 296.76699317912215, "total_per_gpu": 591.3419528941088, "tpot_error_pct": 4.86, "tpot_ms": 6.551778306956284, "tpot_relative": 1.043246, "ttft_error_pct": 34.03, "ttft_ms": 39.90146192499998, "ttft_relative": 0.801787, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 6453.255574874947, "interactivity_tok_s": 145.2099421789071, "output_per_gpu": 281.55844740888034, "total_per_gpu": 561.0371974355864, "tpot_ms": 6.88658080152626, "tpot_relative": 1.096557, "ttft_ms": 60.48430263745104, "ttft_relative": 1.215382, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.06, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.16, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.86, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.26, "ttft_shape_error_pct": null}, "framework": "trt", "id": "66fa7cb4b09daa4c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5821.868674790154, "interactivity_tok_s": 177.45666637774534, "output_per_gpu": 87.62987407685544, "total_per_gpu": null, "tpot_error_pct": 0.06, "tpot_ms": 5.635178550414881, "tpot_relative": 1.000605, "ttft_error_pct": 43.16, "ttft_ms": 57.08101771573088, "ttft_relative": 1.431616, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5239.262091274985, "interactivity_tok_s": 176.05012530321866, "output_per_gpu": 84.73356812111436, "total_per_gpu": 170.29579817678103, "tpot_error_pct": 0.86, "tpot_ms": 5.680200444490779, "tpot_relative": 1.008599, "ttft_error_pct": 20.26, "ttft_ms": 31.794897524999726, "ttft_relative": 0.797429, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 5202.839323850139, "interactivity_tok_s": 177.56406324681433, "output_per_gpu": 85.45733443090018, "total_per_gpu": 171.75040895443956, "tpot_ms": 5.631770200088282, "tpot_relative": 1.0, "ttft_ms": 39.87175045017466, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.88, "tpot_shape_error_pct": 3.24, "ttft_mape_pct": 108.59, "ttft_shape_error_pct": 9.64}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.8, "tpot_shape_error_pct": 2.91, "ttft_mape_pct": 31.92, "ttft_shape_error_pct": 1.49}, "framework": "vllm", "id": "0e631649a5510aaf", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6489.7056411058975, "interactivity_tok_s": 159.90473607040545, "output_per_gpu": 157.6342682663913, "total_per_gpu": null, "tpot_error_pct": 19.45, "tpot_ms": 6.253723464198733, "tpot_relative": 1.19455, "ttft_error_pct": 122.91, "ttft_ms": 92.14653723059355, "ttft_relative": 2.229106, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5745.394097274161, "interactivity_tok_s": 160.3589323785063, "output_per_gpu": 154.6641362492796, "total_per_gpu": 310.8408286813343, "tpot_error_pct": 19.12, "tpot_ms": 6.236010586798064, "tpot_relative": 1.191166, "ttft_error_pct": 31.23, "ttft_ms": 28.426649624999833, "ttft_relative": 0.687666, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 4840.7333642593585, "interactivity_tok_s": 191.01414524598505, "output_per_gpu": 182.72289324965607, "total_per_gpu": 367.232746609275, "tpot_ms": 5.235214380129888, "tpot_relative": 1.0, "ttft_ms": 41.33789120824076, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7280.610854706979, "interactivity_tok_s": 142.49468565434336, "output_per_gpu": 281.0203760138125, "total_per_gpu": null, "tpot_error_pct": 17.36, "tpot_ms": 7.0178055792603455, "tpot_relative": 1.3405, "ttft_error_pct": 114.21, "ttft_ms": 101.39574712364525, "ttft_relative": 2.452852, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6540.37563816247, "interactivity_tok_s": 142.6386643840544, "output_per_gpu": 277.97367469003893, "total_per_gpu": 553.894130487566, "tpot_error_pct": 17.24, "tpot_ms": 7.010721842623971, "tpot_relative": 1.339147, "ttft_error_pct": 31.47, "ttft_ms": 32.437664637499935, "ttft_relative": 0.784696, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5597.731269712858, "interactivity_tok_s": 167.23158836971197, "output_per_gpu": 323.28369676081843, "total_per_gpu": 644.1794976369978, "tpot_ms": 5.979731519318118, "tpot_relative": 1.142213, "ttft_ms": 47.33487913817953, "ttft_relative": 1.145072, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8095.00516368205, "interactivity_tok_s": 128.12638568550742, "output_per_gpu": 505.4968980574109, "total_per_gpu": null, "tpot_error_pct": 13.81, "tpot_ms": 7.804793639106856, "tpot_relative": 1.490826, "ttft_error_pct": 88.64, "ttft_ms": 110.70127087573589, "ttft_relative": 2.677961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7196.4382786687365, "interactivity_tok_s": 127.85574870015834, "output_per_gpu": 497.5834461292716, "total_per_gpu": 1000.4638370185226, "tpot_error_pct": 14.05, "tpot_ms": 7.82131433405592, "tpot_relative": 1.493982, "ttft_error_pct": 33.05, "ttft_ms": 39.28957195624996, "ttft_relative": 0.950449, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 6333.855323674288, "interactivity_tok_s": 145.8226650865339, "output_per_gpu": 563.5003746422858, "total_per_gpu": 1132.9994021335099, "tpot_ms": 6.857644519160181, "tpot_relative": 1.309907, "ttft_ms": 58.68472096808546, "ttft_relative": 1.419635, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.98, "tpot_shape_error_pct": 0.06, "ttft_mape_pct": 137.2, "ttft_shape_error_pct": 3.55}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.66, "tpot_shape_error_pct": 0.36, "ttft_mape_pct": 32.52, "ttft_shape_error_pct": 0.4}, "framework": "vllm", "id": "47c11fa1c28af432", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6143.981008225127, "interactivity_tok_s": 168.90864383651444, "output_per_gpu": 83.25221046667299, "total_per_gpu": null, "tpot_error_pct": 17.02, "tpot_ms": 5.920360126553934, "tpot_relative": 1.170174, "ttft_error_pct": 141.49, "ttft_ms": 87.45259876045196, "ttft_relative": 2.414905, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5425.815599900041, "interactivity_tok_s": 169.7285040948578, "output_per_gpu": 81.80195898869067, "total_per_gpu": 164.40390989426658, "tpot_error_pct": 16.45, "tpot_ms": 5.891762290210961, "tpot_relative": 1.164521, "ttft_error_pct": 32.39, "ttft_ms": 24.484594625000106, "ttft_relative": 0.676115, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 4674.332285299897, "interactivity_tok_s": 197.6524573729948, "output_per_gpu": 94.27065526380646, "total_per_gpu": 189.46324153199078, "tpot_ms": 5.059385617011963, "tpot_relative": 1.0, "ttft_ms": 36.21367437626759, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6707.316194536632, "interactivity_tok_s": 154.74109811339514, "output_per_gpu": 152.52001997956694, "total_per_gpu": null, "tpot_error_pct": 16.95, "tpot_ms": 6.4624072867002305, "tpot_relative": 1.277311, "ttft_error_pct": 132.91, "ttft_ms": 96.27354024229562, "ttft_relative": 2.658486, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6022.635519900019, "interactivity_tok_s": 154.84234718832772, "output_per_gpu": 150.78968810333376, "total_per_gpu": 300.46558643231225, "tpot_error_pct": 16.87, "tpot_ms": 6.4581816160649215, "tpot_relative": 1.276475, "ttft_error_pct": 32.66, "ttft_ms": 27.8362499125001, "ttft_relative": 0.768667, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5170.70785217511, "interactivity_tok_s": 180.97106697343258, "output_per_gpu": 175.5217106358887, "total_per_gpu": 349.74695140740823, "tpot_ms": 5.525745174209559, "tpot_relative": 1.092177, "ttft_ms": 41.33514462446328, "ttft_relative": 1.141424, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.22, "tpot_shape_error_pct": 4.62, "ttft_mape_pct": 11.21, "ttft_shape_error_pct": 21.7}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.65, "tpot_shape_error_pct": 2.91, "ttft_mape_pct": 29.96, "ttft_shape_error_pct": 23.82}, "framework": "vllm", "id": "261fd7da43c95798", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11514.677149759877, "interactivity_tok_s": 89.81051307401628, "output_per_gpu": 1421.4901370761693, "total_per_gpu": null, "tpot_error_pct": 11.8, "tpot_ms": 11.134553915484947, "tpot_relative": 1.118045, "ttft_error_pct": 3.35, "ttft_ms": 124.02849421877536, "ttft_relative": 1.033456, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10656.021861345393, "interactivity_tok_s": 87.06471585881793, "output_per_gpu": 1343.2330599033874, "total_per_gpu": 2687.0717530502166, "tpot_error_pct": 15.33, "tpot_ms": 11.485709108860771, "tpot_relative": 1.153306, "ttft_error_pct": 37.41, "ttft_ms": 75.11599454843774, "ttft_relative": 0.625897, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 9293.602272603084, "interactivity_tok_s": 100.41223804094123, "output_per_gpu": 1537.1951077587437, "total_per_gpu": 3075.083301837878, "tpot_ms": 9.958945438426225, "tpot_relative": 1.0, "ttft_ms": 120.01331526887498, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13748.52101867243, "interactivity_tok_s": 75.1447253300366, "output_per_gpu": 2381.056111820311, "total_per_gpu": null, "tpot_error_pct": 6.64, "tpot_ms": 13.307653938556395, "tpot_relative": 1.336251, "ttft_error_pct": 19.08, "ttft_ms": 134.7910395292388, "ttft_relative": 1.123134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12976.672961653863, "interactivity_tok_s": 71.56868020057631, "output_per_gpu": 2203.7412189474335, "total_per_gpu": 4412.323702918773, "tpot_error_pct": 11.97, "tpot_ms": 13.972592441238666, "tpot_relative": 1.403019, "ttft_error_pct": 22.5, "ttft_ms": 129.09410217812447, "ttft_relative": 1.075665, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 11639.511000119637, "interactivity_tok_s": 80.13686895407947, "output_per_gpu": 2451.5971144694213, "total_per_gpu": 4908.579993501825, "tpot_ms": 12.47865075154142, "tpot_relative": 1.253009, "ttft_ms": 166.57523466128623, "ttft_relative": 1.387973, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.25, "tpot_shape_error_pct": null, "ttft_mape_pct": 85.03, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.36, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.54, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "1e60fb25dc4c5fe8", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 10218.714365090887, "interactivity_tok_s": 101.3141572495926, "output_per_gpu": 800.8835267925809, "total_per_gpu": null, "tpot_error_pct": 14.25, "tpot_ms": 9.870288883087177, "tpot_relative": 1.142455, "ttft_error_pct": 85.03, "ttft_ms": 121.40883769270425, "ttft_relative": 1.85027, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9092.406363400009, "interactivity_tok_s": 101.21209468628713, "output_per_gpu": 788.661780310354, "total_per_gpu": 1585.7191330962553, "tpot_error_pct": 14.36, "tpot_ms": 9.880242110387687, "tpot_relative": 1.143607, "ttft_error_pct": 22.54, "ttft_ms": 50.82578061875002, "ttft_relative": 0.774585, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 7970.631556437002, "interactivity_tok_s": 115.74681955513991, "output_per_gpu": 896.8703155194435, "total_per_gpu": 1803.2881201186133, "tpot_ms": 8.639546242768391, "tpot_relative": 1.0, "ttft_ms": 65.61681219318416, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.96, "tpot_shape_error_pct": 1.13, "ttft_mape_pct": 87.99, "ttft_shape_error_pct": 11.53}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.51, "tpot_shape_error_pct": 0.81, "ttft_mape_pct": 17.5, "ttft_shape_error_pct": 1.32}, "framework": "vllm", "id": "dbd23a23404ed591", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8226.588659475521, "interactivity_tok_s": 126.07333014262726, "output_per_gpu": 497.4115237045147, "total_per_gpu": null, "tpot_error_pct": 15.71, "tpot_ms": 7.93189169246736, "tpot_relative": 1.157127, "ttft_error_pct": 103.64, "ttft_ms": 112.26345808141178, "ttft_relative": 2.036436, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7263.168574900014, "interactivity_tok_s": 127.02407730762457, "output_per_gpu": 491.60755919810913, "total_per_gpu": 988.0228525690947, "tpot_error_pct": 14.85, "tpot_ms": 7.872523234931426, "tpot_relative": 1.148466, "ttft_error_pct": 17.01, "ttft_ms": 45.74974797499963, "ttft_relative": 0.829891, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 6339.471209351541, "interactivity_tok_s": 145.8828061747959, "output_per_gpu": 561.5770860500055, "total_per_gpu": 1128.646181522549, "tpot_ms": 6.854817412833464, "tpot_relative": 1.0, "ttft_ms": 55.12740862650389, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10252.845777194922, "interactivity_tok_s": 100.99445720359167, "output_per_gpu": 798.2174098632606, "total_per_gpu": null, "tpot_error_pct": 15.9, "tpot_ms": 9.901533486972758, "tpot_relative": 1.444463, "ttft_error_pct": 88.9, "ttft_ms": 123.57702002178941, "ttft_relative": 2.241662, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9195.604110924985, "interactivity_tok_s": 101.54783007428364, "output_per_gpu": 793.4846810270782, "total_per_gpu": 1581.1083835286893, "tpot_error_pct": 15.26, "tpot_ms": 9.84757625316549, "tpot_relative": 1.436592, "ttft_error_pct": 18.84, "ttft_ms": 53.095628199999474, "ttft_relative": 0.963144, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 7996.100012573879, "interactivity_tok_s": 117.04879227850941, "output_per_gpu": 909.6906918302427, "total_per_gpu": 1812.6620635058316, "tpot_ms": 8.543445690755783, "tpot_relative": 1.246342, "ttft_ms": 65.4199660231825, "ttft_relative": 1.186705, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12791.897848629891, "interactivity_tok_s": 80.82549641602431, "output_per_gpu": 1279.5599365854173, "total_per_gpu": null, "tpot_error_pct": 13.28, "tpot_ms": 12.372333537585819, "tpot_relative": 1.804911, "ttft_error_pct": 71.43, "ttft_ms": 135.0006396795988, "ttft_relative": 2.448884, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11400.922685200001, "interactivity_tok_s": 80.73393462961357, "output_per_gpu": 1260.1308309107555, "total_per_gpu": 2533.676182448363, "tpot_error_pct": 13.41, "tpot_ms": 12.386365220371601, "tpot_relative": 1.806958, "ttft_error_pct": 16.65, "ttft_ms": 65.63641001874973, "ttft_relative": 1.190631, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 10073.389554643745, "interactivity_tok_s": 91.556325088021, "output_per_gpu": 1423.366949699913, "total_per_gpu": 2861.8861239450575, "tpot_ms": 10.922238294717637, "tpot_relative": 1.593367, "ttft_ms": 78.75025733774237, "ttft_relative": 1.428514, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.67, "tpot_shape_error_pct": null, "ttft_mape_pct": 12.54, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.73, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.33, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "26d8adf1e61625b8", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 18629.176079425808, "interactivity_tok_s": 55.36111461359881, "output_per_gpu": 3514.487152886366, "total_per_gpu": null, "tpot_error_pct": 7.67, "tpot_ms": 18.06322013166913, "tpot_relative": 1.076743, "ttft_error_pct": 12.54, "ttft_ms": 150.50188472828881, "ttft_relative": 0.874607, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17559.79208963325, "interactivity_tok_s": 52.87834255799636, "output_per_gpu": 3264.927361158297, "total_per_gpu": 6537.027242621279, "tpot_error_pct": 12.73, "tpot_ms": 18.911334047643635, "tpot_relative": 1.127299, "ttft_error_pct": 0.33, "ttft_ms": 171.50543963984336, "ttft_relative": 0.996664, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 15593.294238913393, "interactivity_tok_s": 59.609717246835956, "output_per_gpu": 3670.3451786620685, "total_per_gpu": 7348.753515369348, "tpot_ms": 16.77578834771405, "tpot_relative": 1.0, "ttft_ms": 172.07953118514752, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 66.67, "points": 2, "status_counts": {"failed": 1, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.28, "tpot_shape_error_pct": 6.48, "ttft_mape_pct": 67.89, "ttft_shape_error_pct": 96.74}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.21, "tpot_shape_error_pct": 22.4, "ttft_mape_pct": 70.19, "ttft_shape_error_pct": 95.52}, "framework": "vllm", "id": "4150a0dfd4f2bce1", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 22978.714546692398, "interactivity_tok_s": 44.83670816332577, "output_per_gpu": 5698.491085474942, "total_per_gpu": null, "tpot_error_pct": 5.21, "tpot_ms": 22.303153843438288, "tpot_relative": 0.947917, "ttft_error_pct": 37.81, "ttft_ms": 162.58816485502805, "ttft_relative": 0.621893, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23573.66695740899, "interactivity_tok_s": 39.46904933326138, "output_per_gpu": 4880.661476398521, "total_per_gpu": 9756.605809310762, "tpot_error_pct": 7.68, "tpot_ms": 25.33630824386944, "tpot_relative": 1.07683, "ttft_error_pct": 17.92, "ttft_ms": 214.5775680988282, "ttft_relative": 0.820751, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 21947.301086890497, "interactivity_tok_s": 42.50147137885523, "output_per_gpu": 5232.143874772054, "total_per_gpu": 10459.23089945985, "tpot_ms": 23.52859718869655, "tpot_relative": 1.0, "ttft_ms": 261.44059708693135, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30942.16494586178, "interactivity_tok_s": 33.26064871954173, "output_per_gpu": 8463.790444470016, "total_per_gpu": null, "tpot_error_pct": 11.35, "tpot_ms": 30.065559106562677, "tpot_relative": 1.27783, "ttft_error_pct": 97.97, "ttft_ms": 185.09797984816066, "ttft_relative": 0.707992, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32419.847400621482, "interactivity_tok_s": 28.809294148588137, "output_per_gpu": 7121.925337844043, "total_per_gpu": 14237.290107271985, "tpot_error_pct": 2.35, "tpot_ms": 34.711020507561, "tpot_relative": 1.475269, "ttft_error_pct": 95.5, "ttft_ms": 410.1547065824226, "ttft_relative": 1.568826, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 40387.16430352258, "interactivity_tok_s": 29.486227588619194, "output_per_gpu": 5675.462171879018, "total_per_gpu": 11345.69622690686, "tpot_ms": 33.91413828691908, "tpot_relative": 1.441401, "ttft_ms": 9118.612741453602, "ttft_relative": 34.878335, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 48483.278798088766, "interactivity_tok_s": 19.562831227822837, "output_per_gpu": 9495.388786374562, "total_per_gpu": 18990.218347409227, "tpot_error_pct": 50.59, "tpot_ms": 51.11734535529655, "tpot_relative": 2.172562, "ttft_error_pct": 97.15, "ttft_ms": 1367.8443849909083, "ttft_relative": 5.231951, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 79301.54680659976, "interactivity_tok_s": 29.45943307866265, "output_per_gpu": 5695.6952892342, "total_per_gpu": 11391.055133843041, "tpot_ms": 33.944984526002166, "tpot_relative": 1.442712, "ttft_ms": 48019.162936759814, "ttft_relative": 183.671409, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 79.25, "points": 42, "status_counts": {"failed": 11, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.45, "tpot_shape_error_pct": 8.88, "ttft_mape_pct": 60.63, "ttft_shape_error_pct": 34.47}, "aisimulate": {"coverage_pct": 100.0, "points": 53, "status_counts": {"failed": 0, "success": 53, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.65, "tpot_shape_error_pct": 9.46, "ttft_mape_pct": 36.92, "ttft_shape_error_pct": 16.6}, "gpu": "b300", "precisions": ["fp4"], "rows": 53, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.71, "tpot_shape_error_pct": 41.27, "ttft_mape_pct": 95.74, "ttft_shape_error_pct": 70.97}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.92, "tpot_shape_error_pct": 30.91, "ttft_mape_pct": 46.2, "ttft_shape_error_pct": 37.68}, "framework": "dynamo-vllm", "id": "6b707102536986de", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 19484.262, "interactivity_tok_s": 52.700922266139656, "output_per_gpu": 4555.1002, "total_per_gpu": null, "tpot_error_pct": 12.04, "tpot_ms": 18.975, "tpot_relative": 1.120357, "ttft_error_pct": 90.89, "ttft_ms": 72.837, "ttft_relative": 0.091134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16542.810990249498, "interactivity_tok_s": 57.76053142867823, "output_per_gpu": 5495.15662522186, "total_per_gpu": 10989.989616425077, "tpot_error_pct": 2.22, "tpot_ms": 17.312860101274932, "tpot_relative": 1.022217, "ttft_error_pct": 25.0, "ttft_ms": 599.3858563133778, "ttft_relative": 0.749951, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 16393.249915951565, "interactivity_tok_s": 59.043823255223046, "output_per_gpu": 6903.3265803320555, "total_per_gpu": 11066.984059829416, "tpot_ms": 16.936572614503575, "tpot_relative": 1.0, "ttft_ms": 799.2335030937512, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24698.493, "interactivity_tok_s": 41.542040545031576, "output_per_gpu": 4555.1002, "total_per_gpu": null, "tpot_error_pct": 20.27, "tpot_ms": 24.072, "tpot_relative": 1.421303, "ttft_error_pct": 94.14, "ttft_ms": 72.837, "ttft_relative": 0.091134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20992.529195448125, "interactivity_tok_s": 45.868316135215224, "output_per_gpu": 6518.186007058912, "total_per_gpu": 13033.304039547838, "tpot_error_pct": 8.93, "tpot_ms": 21.801541548900545, "tpot_relative": 1.287246, "ttft_error_pct": 26.84, "ttft_ms": 909.6638390084669, "ttft_relative": 1.13817, "unavailable_metrics": {}}, "concurrency": 1536, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 19683.13428627858, "interactivity_tok_s": 49.964612797131075, "output_per_gpu": 8644.053151027992, "total_per_gpu": 13851.164833730252, "tpot_ms": 20.01416490627581, "tpot_relative": 1.181713, "ttft_ms": 1243.3639813132834, "ttft_relative": 1.555696, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30141.876, "interactivity_tok_s": 34.02170584833124, "output_per_gpu": 4555.1002, "total_per_gpu": null, "tpot_error_pct": 37.5, "tpot_ms": 29.393, "tpot_relative": 1.735475, "ttft_error_pct": 98.2, "ttft_ms": 72.837, "ttft_relative": 0.091134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25546.12944370602, "interactivity_tok_s": 38.05331007356528, "output_per_gpu": 7164.59997785665, "total_per_gpu": 14326.058587790587, "tpot_error_pct": 22.93, "tpot_ms": 26.27892286023959, "tpot_relative": 1.551608, "ttft_error_pct": 67.09, "ttft_ms": 1333.6861966349868, "ttft_relative": 1.668707, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 23750.028679876414, "interactivity_tok_s": 46.780301622199524, "output_per_gpu": 9557.134664990876, "total_per_gpu": 15314.83055865284, "tpot_ms": 21.376518862064188, "tpot_relative": 1.262151, "ttft_ms": 4053.1210809659674, "ttft_relative": 5.07126, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48034.146, "interactivity_tok_s": 21.329693065716782, "output_per_gpu": 4555.1002, "total_per_gpu": null, "tpot_error_pct": 117.03, "tpot_ms": 46.883, "tpot_relative": 2.768152, "ttft_error_pct": 99.72, "ttft_ms": 72.837, "ttft_relative": 0.091134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42567.06874179504, "interactivity_tok_s": 27.294316587997148, "output_per_gpu": 8611.17141008711, "total_per_gpu": 17228.600493394522, "tpot_error_pct": 69.6, "tpot_ms": 36.63766399044981, "tpot_relative": 2.163228, "ttft_error_pct": 65.85, "ttft_ms": 8847.236393475465, "ttft_relative": 11.069652, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 45786.90115267776, "interactivity_tok_s": 46.291139635789115, "output_per_gpu": 9860.616770414816, "total_per_gpu": 15812.11234119985, "tpot_ms": 21.602406159533583, "tpot_relative": 1.275489, "ttft_ms": 25903.80762277859, "ttft_relative": 32.410813, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.53, "tpot_shape_error_pct": 9.75, "ttft_mape_pct": 60.76, "ttft_shape_error_pct": 42.33}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.78, "tpot_shape_error_pct": 8.94, "ttft_mape_pct": 34.51, "ttft_shape_error_pct": 4.34}, "framework": "dynamo-vllm", "id": "4ff05e230e54ff1c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9097.743, "interactivity_tok_s": 113.35298118340513, "output_per_gpu": 668.1232, "total_per_gpu": null, "tpot_error_pct": 10.36, "tpot_ms": 8.822, "tpot_relative": 1.10357, "ttft_error_pct": 45.34, "ttft_ms": 72.837, "ttft_relative": 0.546634, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7587.601509109311, "interactivity_tok_s": 122.97547770160867, "output_per_gpu": 755.9919688408985, "total_per_gpu": 1509.592537385884, "tpot_error_pct": 1.72, "tpot_ms": 8.131702504351555, "tpot_relative": 1.017219, "ttft_error_pct": 36.35, "ttft_ms": 84.80931746250045, "ttft_relative": 0.636485, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 7517.0738020469, "interactivity_tok_s": 125.09296652348087, "output_per_gpu": 960.0249448770315, "total_per_gpu": 1535.287205526767, "tpot_ms": 7.994054564309119, "tpot_relative": 1.0, "ttft_ms": 133.24636713752795, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11307.423, "interactivity_tok_s": 91.05809506465125, "output_per_gpu": 1073.2176, "total_per_gpu": null, "tpot_error_pct": 20.3, "tpot_ms": 10.982, "tpot_relative": 1.373771, "ttft_error_pct": 62.69, "ttft_ms": 72.837, "ttft_relative": 0.546634, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9430.917284157731, "interactivity_tok_s": 98.93970892371544, "output_per_gpu": 1209.9155774752462, "total_per_gpu": 2420.376678372563, "tpot_error_pct": 10.72, "tpot_ms": 10.107165372510046, "tpot_relative": 1.264335, "ttft_error_pct": 36.11, "ttft_ms": 124.72705729687456, "ttft_relative": 0.936063, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 8597.399113857795, "interactivity_tok_s": 109.54641364179824, "output_per_gpu": 1666.5195028079988, "total_per_gpu": 2673.1092188329794, "tpot_ms": 9.128550782774715, "tpot_relative": 1.141917, "ttft_ms": 195.23285230936978, "ttft_relative": 1.465202, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13440.378, "interactivity_tok_s": 76.52865998316369, "output_per_gpu": 1804.2716, "total_per_gpu": null, "tpot_error_pct": 21.93, "tpot_ms": 13.067, "tpot_relative": 1.63459, "ttft_error_pct": 74.26, "ttft_ms": 72.837, "ttft_relative": 0.546634, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11116.938740786716, "interactivity_tok_s": 84.13257444804537, "output_per_gpu": 2047.0735237953515, "total_per_gpu": 4098.644138885682, "tpot_error_pct": 10.91, "tpot_ms": 11.886002616234368, "tpot_relative": 1.486855, "ttft_error_pct": 31.07, "ttft_ms": 195.07739009921892, "ttft_relative": 1.464035, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 10131.596412132041, "interactivity_tok_s": 93.30852874711735, "output_per_gpu": 2822.231096173105, "total_per_gpu": 4530.9670750502555, "tpot_ms": 10.717133936493386, "tpot_relative": 1.340638, "ttft_ms": 283.0067358640761, "ttft_relative": 2.123936, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.43, "tpot_shape_error_pct": 12.98, "ttft_mape_pct": 32.73, "ttft_shape_error_pct": 12.56}, "framework": "dynamo-vllm", "id": "1252ec244f92d706", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26873.200156510284, "interactivity_tok_s": 35.86338711413729, "output_per_gpu": 8480.2975197948, "total_per_gpu": 16956.876795069864, "tpot_error_pct": 0.63, "tpot_ms": 27.883590493486924, "tpot_relative": 1.006289, "ttft_error_pct": 28.59, "ttft_ms": 1182.8806151496472, "ttft_relative": 0.714119, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 27188.461451102958, "interactivity_tok_s": 36.08892311629286, "output_per_gpu": 11178.879200563148, "total_per_gpu": 16793.995414828583, "tpot_ms": 27.709333325840795, "tpot_relative": 1.0, "ttft_ms": 1656.4196507739452, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42606.98629356812, "interactivity_tok_s": 27.031325173305298, "output_per_gpu": 10718.164613075096, "total_per_gpu": 21444.118035416195, "tpot_error_pct": 13.22, "tpot_ms": 36.99411677336289, "tpot_relative": 1.335078, "ttft_error_pct": 49.41, "ttft_ms": 8558.425460180839, "ttft_relative": 5.166822, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 46992.60599164784, "interactivity_tok_s": 30.603755738301672, "output_per_gpu": 12901.665507762154, "total_per_gpu": 19395.58420385101, "tpot_ms": 32.675728056098194, "tpot_relative": 1.179232, "ttft_ms": 16918.291078481816, "ttft_relative": 10.213771, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 62355.984111021724, "interactivity_tok_s": 26.630760366524488, "output_per_gpu": 10940.596419913605, "total_per_gpu": 21884.184712959493, "tpot_error_pct": 13.72, "tpot_ms": 37.55056131469022, "tpot_relative": 1.355159, "ttft_error_pct": 29.5, "ttft_ms": 27788.59296175384, "ttft_relative": 16.7763, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 69812.81359958934, "interactivity_tok_s": 30.28416932731101, "output_per_gpu": 12964.796177742295, "total_per_gpu": 19485.718712072605, "tpot_ms": 33.020552394619436, "tpot_relative": 1.191676, "ttft_ms": 39415.86655280192, "ttft_relative": 23.795822, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 82051.86691493563, "interactivity_tok_s": 26.43506826476069, "output_per_gpu": 11054.619636124475, "total_per_gpu": 22111.556177204395, "tpot_error_pct": 14.14, "tpot_ms": 37.82853859065125, "tpot_relative": 1.365191, "ttft_error_pct": 23.43, "ttft_ms": 47227.80192302413, "ttft_relative": 28.511979, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 92182.53928182945, "interactivity_tok_s": 30.17401267345637, "output_per_gpu": 13049.806375115824, "total_per_gpu": 19614.28852525767, "tpot_ms": 33.14110094742835, "tpot_relative": 1.196027, "ttft_ms": 61675.83512499023, "ttft_relative": 37.234426, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.62, "tpot_shape_error_pct": 7.5, "ttft_mape_pct": 29.39, "ttft_shape_error_pct": 11.04}, "framework": "dynamo-vllm", "id": "28925bd14c12b0ff", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7366.185865985937, "interactivity_tok_s": 127.17990059675394, "output_per_gpu": 592.6179234835255, "total_per_gpu": 1185.5030449134347, "tpot_error_pct": 0.94, "tpot_ms": 7.862877666264849, "tpot_relative": 1.00938, "ttft_error_pct": 34.79, "ttft_ms": 126.20840713124907, "ttft_relative": 0.652098, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 7363.350578492162, "interactivity_tok_s": 128.3728215957623, "output_per_gpu": 647.3954569431838, "total_per_gpu": 1198.1849421477461, "tpot_ms": 7.789810861592925, "tpot_relative": 1.0, "ttft_ms": 193.54198189999323, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8470.656366409381, "interactivity_tok_s": 111.08351517102103, "output_per_gpu": 1030.7201641001075, "total_per_gpu": 2063.7046546270158, "tpot_error_pct": 3.88, "tpot_ms": 9.00223582644489, "tpot_relative": 1.155642, "ttft_error_pct": 31.85, "ttft_ms": 198.54219962812482, "ttft_relative": 1.025835, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 8254.985888251567, "interactivity_tok_s": 115.389437976017, "output_per_gpu": 1152.9034321555955, "total_per_gpu": 2135.6947123089853, "tpot_ms": 8.666304451607122, "tpot_relative": 1.112518, "ttft_ms": 291.34751155234113, "ttft_relative": 1.505345, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10530.590485304303, "interactivity_tok_s": 90.41851052033816, "output_per_gpu": 1662.8992968997577, "total_per_gpu": 3324.191407022755, "tpot_error_pct": 11.42, "tpot_ms": 11.059682295640851, "tpot_relative": 1.419763, "ttft_error_pct": 22.98, "ttft_ms": 346.02828506601634, "ttft_relative": 1.787872, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 9591.137761380069, "interactivity_tok_s": 100.74115186240256, "output_per_gpu": 1970.0814479159737, "total_per_gpu": 3642.5137478064466, "tpot_ms": 9.926430078602351, "tpot_relative": 1.274284, "ttft_ms": 449.24522044217844, "ttft_relative": 2.321177, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20819.124227876986, "interactivity_tok_s": 78.39328086107993, "output_per_gpu": 3339.1134473389175, "total_per_gpu": 6678.030239554538, "tpot_error_pct": 10.24, "tpot_ms": 12.75619528888058, "tpot_relative": 1.637549, "ttft_error_pct": 27.95, "ttft_ms": 9071.615638600128, "ttft_relative": 46.871565, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 23245.686770322267, "interactivity_tok_s": 86.41748098958467, "output_per_gpu": 3236.372108741604, "total_per_gpu": 5986.558373593622, "tpot_ms": 11.571732808556678, "tpot_relative": 1.485496, "ttft_ms": 12591.127715681543, "ttft_relative": 65.056313, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.75, "tpot_shape_error_pct": null, "ttft_mape_pct": 11.87, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.21, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.49, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "161bc5b0e30c18c0", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6261.987, "interactivity_tok_s": 165.28925619834712, "output_per_gpu": 67.62044444444444, "total_per_gpu": null, "tpot_error_pct": 13.75, "tpot_ms": 6.05, "tpot_relative": 1.137481, "ttft_error_pct": 11.87, "ttft_ms": 72.837, "ttft_relative": 0.881315, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5176.831330775088, "interactivity_tok_s": 178.6950094456111, "output_per_gpu": 76.22470725223953, "total_per_gpu": 153.19486302946078, "tpot_error_pct": 5.21, "tpot_ms": 5.596127183979176, "tpot_relative": 1.052147, "ttft_error_pct": 43.49, "ttft_ms": 46.70696949999943, "ttft_relative": 0.565146, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 4958.459041977767, "interactivity_tok_s": 188.01344956867763, "output_per_gpu": 89.30710427605237, "total_per_gpu": 160.15003820126728, "tpot_ms": 5.318768430099569, "tpot_relative": 1.0, "ttft_ms": 82.64581586117856, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.77, "tpot_shape_error_pct": null, "ttft_mape_pct": 17.84, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "f310bd89b36a8472", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 81124.42242531796, "interactivity_tok_s": 36.82294254459926, "output_per_gpu": 8899.032361345484, "total_per_gpu": 17804.531582290227, "tpot_error_pct": 11.77, "tpot_ms": 27.156982329394744, "tpot_relative": 1.11767, "ttft_error_pct": 17.84, "ttft_ms": 56128.732430967895, "ttft_relative": 0.821559, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 90683.97472023028, "interactivity_tok_s": 41.155893339052824, "output_per_gpu": 9934.719712462327, "total_per_gpu": 15930.941018122398, "tpot_ms": 24.297856731276443, "tpot_relative": 1.0, "ttft_ms": 68319.82275848147, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.46, "tpot_shape_error_pct": 2.69, "ttft_mape_pct": 19.34, "ttft_shape_error_pct": 24.26}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.36, "tpot_shape_error_pct": 2.36, "ttft_mape_pct": 41.0, "ttft_shape_error_pct": 0.01}, "framework": "dynamo-vllm", "id": "dfe7fdd7d7e77f12", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6710.061, "interactivity_tok_s": 154.13070283600493, "output_per_gpu": 113.61479999999999, "total_per_gpu": null, "tpot_error_pct": 16.89, "tpot_ms": 6.488, "tpot_relative": 1.16887, "ttft_error_pct": 8.21, "ttft_ms": 72.837, "ttft_relative": 0.917944, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5546.752554549985, "interactivity_tok_s": 166.67935152245735, "output_per_gpu": 128.18182536722435, "total_per_gpu": 257.6172200309927, "tpot_error_pct": 8.09, "tpot_ms": 5.999543379944493, "tpot_relative": 1.08087, "ttft_error_pct": 41.0, "ttft_ms": 46.814769549999866, "ttft_relative": 0.589993, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5167.724574325257, "interactivity_tok_s": 180.15876767886976, "output_per_gpu": 171.6765878030346, "total_per_gpu": 277.07326405904456, "tpot_ms": 5.550659636962463, "tpot_relative": 1.0, "ttft_ms": 79.34797247517054, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8317.194, "interactivity_tok_s": 124.08487405385284, "output_per_gpu": 365.7154, "total_per_gpu": null, "tpot_error_pct": 20.03, "tpot_ms": 8.059, "tpot_relative": 1.4519, "ttft_error_pct": 30.48, "ttft_ms": 72.837, "ttft_relative": 0.917944, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6858.484649687501, "interactivity_tok_s": 134.61859566788246, "output_per_gpu": 417.16868684658203, "total_per_gpu": 838.7782760322773, "tpot_error_pct": 10.64, "tpot_ms": 7.428394235125585, "tpot_relative": 1.33829, "ttft_error_pct": 41.0, "ttft_ms": 61.816924956250524, "ttft_relative": 0.779061, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 6242.710983812594, "interactivity_tok_s": 148.93910153442977, "output_per_gpu": 571.8818283115172, "total_per_gpu": 921.4507914259113, "tpot_ms": 6.714153568120143, "tpot_relative": 1.209614, "ttft_ms": 104.76671509375137, "ttft_relative": 1.320345, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.77, "tpot_shape_error_pct": 3.27, "ttft_mape_pct": 24.28, "ttft_shape_error_pct": 12.94}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.75, "tpot_shape_error_pct": 2.63, "ttft_mape_pct": 40.29, "ttft_shape_error_pct": 5.18}, "framework": "dynamo-vllm", "id": "04a5ca50ab8de533", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6710.061, "interactivity_tok_s": 154.13070283600493, "output_per_gpu": 126.23866666666666, "total_per_gpu": null, "tpot_error_pct": 14.89, "tpot_ms": 6.488, "tpot_relative": 1.148889, "ttft_error_pct": 19.04, "ttft_ms": 72.837, "ttft_relative": 0.809563, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5626.868207499983, "interactivity_tok_s": 166.49888082568836, "output_per_gpu": 143.22518088760575, "total_per_gpu": 285.3924463175346, "tpot_error_pct": 6.35, "tpot_ms": 6.006046377254174, "tpot_relative": 1.063545, "ttft_error_pct": 41.8, "ttft_ms": 52.366598724999406, "ttft_relative": 0.58204, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5316.908908999937, "interactivity_tok_s": 177.07907304669467, "output_per_gpu": 167.0175973229052, "total_per_gpu": 297.1064768148705, "tpot_ms": 5.647194684243158, "tpot_relative": 1.0, "ttft_ms": 89.97074104997864, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7557.105, "interactivity_tok_s": 136.6867140513942, "output_per_gpu": 223.79633333333334, "total_per_gpu": null, "tpot_error_pct": 18.64, "tpot_ms": 7.316, "tpot_relative": 1.29551, "ttft_error_pct": 29.52, "ttft_ms": 72.837, "ttft_relative": 0.809563, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6221.360287725011, "interactivity_tok_s": 148.5798171836383, "output_per_gpu": 255.18776636707688, "total_per_gpu": 513.0920931671504, "tpot_error_pct": 9.15, "tpot_ms": 6.730389220791965, "tpot_relative": 1.191811, "ttft_error_pct": 38.78, "ttft_ms": 63.2707401750003, "ttft_relative": 0.703237, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5740.983449425022, "interactivity_tok_s": 162.17161665360697, "output_per_gpu": 309.92275398842554, "total_per_gpu": 554.85154994871, "tpot_ms": 6.166307154327541, "tpot_relative": 1.091924, "ttft_ms": 103.34713611249526, "ttft_relative": 1.148675, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.5, "tpot_shape_error_pct": 2.0, "ttft_mape_pct": 77.76, "ttft_shape_error_pct": 28.91}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 47.3, "tpot_shape_error_pct": 5.24, "ttft_mape_pct": 57.02, "ttft_shape_error_pct": 6.89}, "framework": "trt", "id": "eb61b30760a20bea", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 34578.50390792879, "interactivity_tok_s": 29.76577275604247, "output_per_gpu": 7600.039119934522, "total_per_gpu": null, "tpot_error_pct": 19.31, "tpot_ms": 33.59563375679536, "tpot_relative": 1.193114, "ttft_error_pct": 74.0, "ttft_ms": 210.17057472714237, "ttft_relative": 0.259982, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 39611.42205852204, "interactivity_tok_s": 23.47892288188443, "output_per_gpu": 5844.436899757438, "total_per_gpu": 11688.529594839818, "tpot_error_pct": 51.26, "tpot_ms": 42.591391650745926, "tpot_relative": 1.512589, "ttft_error_pct": 58.45, "ttft_ms": 335.85291224267723, "ttft_relative": 0.415451, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 26758.853746886038, "interactivity_tok_s": 35.513957715570804, "output_per_gpu": 8573.355920986995, "total_per_gpu": 17146.206919217606, "tpot_ms": 28.157943082799758, "tpot_relative": 1.0, "ttft_ms": 808.4051245516889, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 54424.49152822845, "interactivity_tok_s": 18.88650164196317, "output_per_gpu": 9654.164853643002, "total_per_gpu": null, "tpot_error_pct": 21.69, "tpot_ms": 52.94786821600351, "tpot_relative": 1.880388, "ttft_error_pct": 81.52, "ttft_ms": 258.8223432568539, "ttft_relative": 0.320164, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 58155.84529804491, "interactivity_tok_s": 16.035000363083903, "output_per_gpu": 7985.066599932741, "total_per_gpu": 15966.63209273385, "tpot_error_pct": 43.33, "tpot_ms": 62.36357825736131, "tpot_relative": 2.214777, "ttft_error_pct": 55.59, "ttft_ms": 621.9351984310581, "ttft_relative": 0.769336, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 41508.275671223586, "interactivity_tok_s": 22.983711113391795, "output_per_gpu": 11112.64750831756, "total_per_gpu": 22220.42260023686, "tpot_ms": 43.509074538329685, "tpot_relative": 1.545179, "ttft_ms": 1400.476266373272, "ttft_relative": 1.732394, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.51, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.36, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.09, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.28, "ttft_shape_error_pct": null}, "framework": "trt", "id": "aeffdd61b0e98695", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7398.29464613397, "interactivity_tok_s": 139.19388060525003, "output_per_gpu": 69.0604339420053, "total_per_gpu": null, "tpot_error_pct": 30.51, "tpot_ms": 7.1842238728581185, "tpot_relative": 1.305076, "ttft_error_pct": 16.36, "ttft_ms": 48.83362420011411, "ttft_relative": 1.163586, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6644.841184300023, "interactivity_tok_s": 138.57592689253968, "output_per_gpu": 66.83293430077471, "total_per_gpu": 134.319469173997, "tpot_error_pct": 31.09, "tpot_ms": 7.216260590307735, "tpot_relative": 1.310896, "ttft_error_pct": 30.28, "ttft_ms": 29.259152499999544, "ttft_relative": 0.697174, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 5088.490932298009, "interactivity_tok_s": 181.65865176370897, "output_per_gpu": 87.39296203027295, "total_per_gpu": 175.64059385184692, "tpot_ms": 5.504830022083077, "tpot_relative": 1.0, "ttft_ms": 41.96820032375399, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.4, "tpot_shape_error_pct": 1.19, "ttft_mape_pct": 14.3, "ttft_shape_error_pct": 8.49}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.0, "tpot_shape_error_pct": 1.02, "ttft_mape_pct": 31.7, "ttft_shape_error_pct": 7.01}, "framework": "trt", "id": "c8c98d5f54c60d02", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7762.83675848629, "interactivity_tok_s": 132.67488584194243, "output_per_gpu": 131.62668998971677, "total_per_gpu": null, "tpot_error_pct": 25.14, "tpot_ms": 7.5372214843381515, "tpot_relative": 1.25145, "ttft_error_pct": 19.37, "ttft_ms": 52.25918000836088, "ttft_relative": 1.193696, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6968.190763074992, "interactivity_tok_s": 132.1511497572863, "output_per_gpu": 127.60415759911497, "total_per_gpu": 256.45623512462623, "tpot_error_pct": 25.64, "tpot_ms": 7.567092695270809, "tpot_relative": 1.25641, "ttft_error_pct": 29.22, "ttft_ms": 30.98731082499957, "ttft_relative": 0.707807, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 5565.39632055501, "interactivity_tok_s": 166.03597425809028, "output_per_gpu": 159.9872296676348, "total_per_gpu": 321.53907333866607, "tpot_ms": 6.022791172023818, "tpot_relative": 1.0, "ttft_ms": 43.77929190522991, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8631.815902423405, "interactivity_tok_s": 119.30903249217718, "output_per_gpu": 236.96460826586662, "total_per_gpu": null, "tpot_error_pct": 23.66, "tpot_ms": 8.381595082212804, "tpot_relative": 1.391646, "ttft_error_pct": 9.23, "ttft_ms": 57.44413331970703, "ttft_relative": 1.31213, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7859.3485910000145, "interactivity_tok_s": 118.63272184767602, "output_per_gpu": 231.3308129008702, "total_per_gpu": 460.9529287605626, "tpot_error_pct": 24.36, "tpot_ms": 8.429377531133412, "tpot_relative": 1.39958, "ttft_error_pct": 34.18, "ttft_ms": 34.61468480000024, "ttft_relative": 0.790663, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 6344.460218897439, "interactivity_tok_s": 147.53542626016915, "output_per_gpu": 286.4606020985904, "total_per_gpu": 570.8052976429182, "tpot_ms": 6.77803308228198, "tpot_relative": 1.125397, "ttft_ms": 52.59004572289996, "ttft_relative": 1.201254, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.5, "tpot_shape_error_pct": 2.55, "ttft_mape_pct": 72.76, "ttft_shape_error_pct": 6.48}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.13, "tpot_shape_error_pct": 2.38, "ttft_mape_pct": 2.92, "ttft_shape_error_pct": 3.51}, "framework": "trt", "id": "b4ba64a8b6c41569", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 10987.746929172437, "interactivity_tok_s": 93.87857191930176, "output_per_gpu": 371.71376045753937, "total_per_gpu": null, "tpot_error_pct": 46.37, "tpot_ms": 10.65205807412156, "tpot_relative": 1.463692, "ttft_error_pct": 78.55, "ttft_ms": 90.69151934608064, "ttft_relative": 1.785471, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9852.24315007501, "interactivity_tok_s": 93.55770053362085, "output_per_gpu": 362.2746474546213, "total_per_gpu": 728.092202600441, "tpot_error_pct": 46.87, "tpot_ms": 10.688591043776675, "tpot_relative": 1.468712, "ttft_error_pct": 4.76, "ttft_ms": 53.212631850000264, "ttft_relative": 1.047613, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 6722.781100048451, "interactivity_tok_s": 137.4092994295693, "output_per_gpu": 530.8163480244579, "total_per_gpu": 1066.8238772017878, "tpot_ms": 7.277527824909415, "tpot_relative": 1.0, "ttft_ms": 50.79417737724725, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13224.089617822608, "interactivity_tok_s": 77.94697538926935, "output_per_gpu": 618.5149937341998, "total_per_gpu": null, "tpot_error_pct": 42.63, "tpot_ms": 12.829234168561028, "tpot_relative": 1.762856, "ttft_error_pct": 66.98, "ttft_ms": 99.78306338467664, "ttft_relative": 1.964459, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12032.847632987485, "interactivity_tok_s": 77.5414342988256, "output_per_gpu": 606.002759290157, "total_per_gpu": 1207.5293525704346, "tpot_error_pct": 43.38, "tpot_ms": 12.896330962182697, "tpot_relative": 1.772076, "ttft_error_pct": 1.08, "ttft_ms": 60.405014749999985, "ttft_relative": 1.189211, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 8409.587829934026, "interactivity_tok_s": 111.1787069612093, "output_per_gpu": 867.1272272103184, "total_per_gpu": 1727.8495241440382, "tpot_ms": 8.994528065062891, "tpot_relative": 1.235932, "ttft_ms": 59.756664669839665, "ttft_relative": 1.176447, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.44, "tpot_shape_error_pct": 3.73, "ttft_mape_pct": 61.54, "ttft_shape_error_pct": 8.71}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.44, "tpot_shape_error_pct": 16.09, "ttft_mape_pct": 68.0, "ttft_shape_error_pct": 30.28}, "framework": "trt", "id": "42cdc8648c49462d", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 12694.714309705021, "interactivity_tok_s": 81.42198149917627, "output_per_gpu": 1290.0996518082902, "total_per_gpu": null, "tpot_error_pct": 15.78, "tpot_ms": 12.281695699215044, "tpot_relative": 1.157796, "ttft_error_pct": 63.9, "ttft_ms": 130.53960940803117, "ttft_relative": 0.361008, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15285.56366752271, "interactivity_tok_s": 60.56313378623291, "output_per_gpu": 933.4377691716452, "total_per_gpu": 1868.9261509946443, "tpot_error_pct": 55.66, "tpot_ms": 16.51169511025729, "tpot_relative": 1.556559, "ttft_error_pct": 73.93, "ttft_ms": 94.28635611093725, "ttft_relative": 0.26075, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 10113.720305356583, "interactivity_tok_s": 94.27006500857506, "output_per_gpu": 1397.4012959648212, "total_per_gpu": 2797.872457828755, "tpot_ms": 10.607821262338552, "tpot_relative": 1.0, "ttft_ms": 361.5973926141123, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14624.292202025637, "interactivity_tok_s": 70.79186684107827, "output_per_gpu": 2246.282385298143, "total_per_gpu": null, "tpot_error_pct": 9.66, "tpot_ms": 14.125916501748923, "tpot_relative": 1.331651, "ttft_error_pct": 60.19, "ttft_ms": 173.47962073648952, "ttft_relative": 0.479759, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19672.86134627666, "interactivity_tok_s": 47.194477133431796, "output_per_gpu": 1460.5918628696347, "total_per_gpu": 2919.7720684413052, "tpot_error_pct": 64.49, "tpot_ms": 21.188919991055826, "tpot_relative": 1.997481, "ttft_error_pct": 73.02, "ttft_ms": 117.57398996015581, "ttft_relative": 0.325152, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 12306.197848182046, "interactivity_tok_s": 77.6285367671445, "output_per_gpu": 2310.5266427747742, "total_per_gpu": 4618.82016904361, "tpot_ms": 12.881860738913735, "tpot_relative": 1.214374, "ttft_ms": 435.79243385697737, "ttft_relative": 1.205187, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18393.426646988886, "interactivity_tok_s": 56.25797402714407, "output_per_gpu": 3576.621829050426, "total_per_gpu": null, "tpot_error_pct": 14.75, "tpot_ms": 17.775257948633328, "tpot_relative": 1.675675, "ttft_error_pct": 58.65, "ttft_ms": 209.3377655369915, "ttft_relative": 0.578925, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26622.2909600729, "interactivity_tok_s": 34.884071200919195, "output_per_gpu": 2169.530042014124, "total_per_gpu": 4337.0615584054385, "tpot_error_pct": 85.06, "tpot_ms": 28.666378824890426, "tpot_relative": 2.702381, "ttft_error_pct": 67.4, "ttft_ms": 165.072378735547, "ttft_relative": 0.456509, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 14786.689799626833, "interactivity_tok_s": 64.55595362054939, "output_per_gpu": 3846.823750467424, "total_per_gpu": 7690.103887487299, "tpot_ms": 15.490438045077239, "tpot_relative": 1.460285, "ttft_ms": 506.2841166145973, "ttft_relative": 1.400132, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24915.91639833783, "interactivity_tok_s": 41.43011442989455, "output_per_gpu": 5277.508253741577, "total_per_gpu": null, "tpot_error_pct": 21.58, "tpot_ms": 24.137032054114584, "tpot_relative": 2.2754, "ttft_error_pct": 63.42, "ttft_ms": 223.73260697860982, "ttft_relative": 0.618734, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35523.37795708423, "interactivity_tok_s": 26.15836100413542, "output_per_gpu": 3262.1715387628974, "total_per_gpu": 6524.150953850966, "tpot_error_pct": 92.56, "tpot_ms": 38.22869482693921, "tpot_relative": 3.603822, "ttft_error_pct": 57.67, "ttft_ms": 258.919907638961, "ttft_relative": 0.716045, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 18910.824432238634, "interactivity_tok_s": 50.371181972135645, "output_per_gpu": 6037.793697913689, "total_per_gpu": 12075.231803517374, "tpot_ms": 19.852621297494675, "tpot_relative": 1.871508, "ttft_ms": 611.6887295196989, "ttft_relative": 1.691629, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.25, "tpot_shape_error_pct": null, "ttft_mape_pct": 277.44, "ttft_shape_error_pct": null}, "framework": "trt", "id": "a6a6c7e46d652883", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 97133.92323475462, "interactivity_tok_s": 12.178796175850826, "output_per_gpu": 9508.148871823167, "total_per_gpu": 19012.12883066268, "tpot_error_pct": 19.25, "tpot_ms": 82.1099216672077, "tpot_relative": 1.192501, "ttft_error_pct": 277.44, "ttft_ms": 21393.496244627077, "ttft_relative": 3.774385, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 69113.95933868483, "interactivity_tok_s": 14.523221037707419, "output_per_gpu": 13351.97833471367, "total_per_gpu": 26698.102403093548, "tpot_ms": 68.85524894261722, "tpot_relative": 1.0, "ttft_ms": 5668.074341080876, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.56, "tpot_shape_error_pct": 2.27, "ttft_mape_pct": 146.21, "ttft_shape_error_pct": 2.74}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.41, "tpot_shape_error_pct": 2.39, "ttft_mape_pct": 31.52, "ttft_shape_error_pct": 0.78}, "framework": "vllm", "id": "478fb12e5a3999cb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5874.590918127995, "interactivity_tok_s": 176.75010037108297, "output_per_gpu": 87.0698925471725, "total_per_gpu": null, "tpot_error_pct": 12.28, "tpot_ms": 5.657705415162548, "tpot_relative": 1.12284, "ttft_error_pct": 149.63, "ttft_ms": 86.7582784167083, "ttft_relative": 2.496307, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5200.859484475093, "interactivity_tok_s": 177.07843120586244, "output_per_gpu": 85.34723669709076, "total_per_gpu": 171.52913677303292, "tpot_error_pct": 12.08, "tpot_ms": 5.647215153139969, "tpot_relative": 1.120758, "ttft_error_pct": 31.78, "ttft_ms": 23.708695924999574, "ttft_relative": 0.682173, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 4654.113886750565, "interactivity_tok_s": 198.46211539957682, "output_per_gpu": 94.82028944601821, "total_per_gpu": 190.56788510879852, "tpot_ms": 5.038745042027967, "tpot_relative": 1.0, "ttft_ms": 34.75465745104884, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6575.566987634685, "interactivity_tok_s": 157.8688025026171, "output_per_gpu": 155.57593769841378, "total_per_gpu": null, "tpot_error_pct": 14.83, "tpot_ms": 6.334373759396968, "tpot_relative": 1.257133, "ttft_error_pct": 142.78, "ttft_ms": 95.50263177158635, "ttft_relative": 2.747909, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5902.889807924988, "interactivity_tok_s": 157.97839853531755, "output_per_gpu": 153.89410408126838, "total_per_gpu": 306.65148799542675, "tpot_error_pct": 14.75, "tpot_ms": 6.329979346995599, "tpot_relative": 1.256261, "ttft_error_pct": 31.25, "ttft_ms": 27.042482262500364, "ttft_relative": 0.778097, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5159.2195283881665, "interactivity_tok_s": 181.28643068597242, "output_per_gpu": 175.89560661320033, "total_per_gpu": 350.49198162466445, "tpot_ms": 5.516132653812451, "tpot_relative": 1.094743, "ttft_ms": 39.336335324878746, "ttft_relative": 1.131829, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.97, "tpot_shape_error_pct": 6.31, "ttft_mape_pct": 61.06, "ttft_shape_error_pct": 47.75}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.48, "tpot_shape_error_pct": 3.58, "ttft_mape_pct": 20.18, "ttft_shape_error_pct": 11.37}, "framework": "vllm", "id": "cfb123b76285d0a9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 7601.988233775256, "interactivity_tok_s": 136.35991376265386, "output_per_gpu": 269.14011664865825, "total_per_gpu": null, "tpot_error_pct": 2.81, "tpot_ms": 7.333533532006964, "tpot_relative": 1.028095, "ttft_error_pct": 107.76, "ttft_ms": 99.78343053213229, "ttft_relative": 2.077582, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6725.2592801499995, "interactivity_tok_s": 137.03671530902386, "output_per_gpu": 264.9383941591332, "total_per_gpu": 532.4677846271588, "tpot_error_pct": 2.3, "tpot_ms": 7.297314429530479, "tpot_relative": 1.023017, "ttft_error_pct": 26.53, "ttft_ms": 35.28502865000015, "ttft_relative": 0.734666, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 6587.063778850643, "interactivity_tok_s": 140.19093389934417, "output_per_gpu": 269.25821764594065, "total_per_gpu": 541.149677824636, "tpot_ms": 7.133128884910568, "tpot_relative": 1.0, "ttft_ms": 48.02863662553136, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8708.627331046015, "interactivity_tok_s": 118.9696354162043, "output_per_gpu": 469.8788734950379, "total_per_gpu": null, "tpot_error_pct": 5.4, "tpot_ms": 8.40550613189317, "tpot_relative": 1.178376, "ttft_error_pct": 98.29, "ttft_ms": 109.79455811930382, "ttft_relative": 2.286023, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7831.426434412501, "interactivity_tok_s": 119.15594531516368, "output_per_gpu": 464.9928407925868, "total_per_gpu": 926.5510682655395, "tpot_error_pct": 5.23, "tpot_ms": 8.392363447371693, "tpot_relative": 1.176533, "ttft_error_pct": 26.7, "ttft_ms": 40.58882217499969, "ttft_relative": 0.845096, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 7457.788039212392, "interactivity_tok_s": 125.39172408642433, "output_per_gpu": 485.7344127568979, "total_per_gpu": 967.8810070841274, "tpot_ms": 7.97500797828384, "tpot_relative": 1.118024, "ttft_ms": 55.370995974863035, "ttft_relative": 1.152875, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10092.83190472826, "interactivity_tok_s": 102.5774899783527, "output_per_gpu": 810.8725159849322, "total_per_gpu": null, "tpot_error_pct": 3.04, "tpot_ms": 9.748727525025554, "tpot_relative": 1.366683, "ttft_error_pct": 73.13, "ttft_ms": 119.88364662711946, "ttft_relative": 2.496087, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8988.607433200033, "interactivity_tok_s": 102.36793809209452, "output_per_gpu": 797.7587158069642, "total_per_gpu": 1604.0098440570944, "tpot_error_pct": 3.25, "tpot_ms": 9.768683619478178, "tpot_relative": 1.369481, "ttft_error_pct": 28.99, "ttft_ms": 49.17103935624981, "ttft_relative": 1.023786, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 8726.957423880958, "interactivity_tok_s": 105.69119172200831, "output_per_gpu": 820.53569153494, "total_per_gpu": 1649.8062641545755, "tpot_ms": 9.461526393138094, "tpot_relative": 1.32642, "ttft_ms": 69.24510792478031, "ttft_relative": 1.441746, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11449.478916666474, "interactivity_tok_s": 90.3755073883219, "output_per_gpu": 1429.5847102852745, "total_per_gpu": null, "tpot_error_pct": 9.89, "tpot_ms": 11.064944794204468, "tpot_relative": 1.551205, "ttft_error_pct": 41.48, "ttft_ms": 130.04039219530313, "ttft_relative": 2.70756, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10401.494026634336, "interactivity_tok_s": 89.29915280164022, "output_per_gpu": 1384.8618228056916, "total_per_gpu": 2765.3429655124482, "tpot_error_pct": 8.8, "tpot_ms": 11.198314526245229, "tpot_relative": 1.569902, "ttft_error_pct": 27.76, "ttft_ms": 66.39453955937455, "ttft_relative": 1.382395, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 11424.395409306499, "interactivity_tok_s": 81.43679504150683, "output_per_gpu": 1260.6445004510908, "total_per_gpu": 2517.3012526777634, "tpot_ms": 12.279461630216643, "tpot_relative": 1.721469, "ttft_ms": 91.91180808129502, "ttft_relative": 1.913688, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15389.904220201402, "interactivity_tok_s": 67.08872141541583, "output_per_gpu": 2127.1087546490003, "total_per_gpu": null, "tpot_error_pct": 1.7, "tpot_ms": 14.905635088914023, "tpot_relative": 2.089635, "ttft_error_pct": 7.38, "ttft_ms": 141.43952424235582, "ttft_relative": 2.9449, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14096.376116556214, "interactivity_tok_s": 65.83494449328543, "output_per_gpu": 2038.012578048731, "total_per_gpu": 4076.9440496273073, "tpot_error_pct": 0.17, "tpot_ms": 15.189501680251146, "tpot_relative": 2.12943, "ttft_error_pct": 21.91, "ttft_ms": 102.86020621874943, "ttft_relative": 2.141643, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 14099.08659480925, "interactivity_tok_s": 65.950069319198, "output_per_gpu": 2032.94040482123, "total_per_gpu": 4066.797416244731, "tpot_ms": 15.162986336830143, "tpot_relative": 2.125713, "ttft_ms": 131.71883188276752, "ttft_relative": 2.742506, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18785.629524326025, "interactivity_tok_s": 54.90698490592954, "output_per_gpu": 3485.217246258291, "total_per_gpu": null, "tpot_error_pct": 3.35, "tpot_ms": 18.212619063189674, "tpot_relative": 2.553244, "ttft_error_pct": 29.84, "ttft_ms": 154.12022268298765, "ttft_relative": 3.208924, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17663.79079541568, "interactivity_tok_s": 52.57997633239339, "output_per_gpu": 3247.095973185758, "total_per_gpu": 6501.325293984699, "tpot_error_pct": 0.93, "tpot_ms": 19.01864682628093, "tpot_relative": 2.666242, "ttft_error_pct": 19.79, "ttft_ms": 176.18766454843734, "ttft_relative": 3.668388, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 17542.082165192045, "interactivity_tok_s": 53.0685771607167, "output_per_gpu": 3265.3580775622468, "total_per_gpu": 6537.889621643843, "tpot_ms": 18.84354270459387, "tpot_relative": 2.641694, "ttft_ms": 219.66198874129077, "ttft_relative": 4.573563, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24791.600532399756, "interactivity_tok_s": 41.54979539307062, "output_per_gpu": 5281.788879619585, "total_per_gpu": null, "tpot_error_pct": 6.8, "tpot_ms": 24.067507205264672, "tpot_relative": 3.374046, "ttft_error_pct": 54.05, "ttft_ms": 170.54066141399747, "ttft_relative": 3.550812, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24023.66390326791, "interactivity_tok_s": 38.92559555944174, "output_per_gpu": 4798.953557495922, "total_per_gpu": 9593.26894194416, "tpot_error_pct": 0.51, "tpot_ms": 25.69003725255634, "tpot_relative": 3.60151, "ttft_error_pct": 8.68, "ttft_ms": 338.95215104570406, "ttft_relative": 7.057293, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 24165.057920786683, "interactivity_tok_s": 38.72646444799734, "output_per_gpu": 4763.576105799192, "total_per_gpu": 9522.548230743016, "tpot_ms": 25.822135179492562, "tpot_relative": 3.620029, "ttft_ms": 371.1704067401115, "ttft_relative": 7.728106, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35571.36708092345, "interactivity_tok_s": 28.90407260373559, "output_per_gpu": 7362.325979887564, "total_per_gpu": null, "tpot_error_pct": 6.81, "tpot_ms": 34.597200668211684, "tpot_relative": 4.850214, "ttft_error_pct": 76.55, "ttft_ms": 178.43079734290183, "ttft_relative": 3.715092, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37246.17558641509, "interactivity_tok_s": 25.260550183466847, "output_per_gpu": 6215.753341650613, "total_per_gpu": 12425.780861543617, "tpot_error_pct": 6.63, "tpot_ms": 39.587419622178494, "tpot_relative": 5.549797, "ttft_error_pct": 1.11, "ttft_ms": 752.5578064357426, "ttft_relative": 15.66894, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 34974.74210749942, "interactivity_tok_s": 26.934795169412084, "output_per_gpu": 6598.833537540225, "total_per_gpu": 13191.588367871958, "tpot_ms": 37.12669777922159, "tpot_relative": 5.204826, "ttft_ms": 760.9727864181423, "ttft_relative": 15.844147, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.49, "tpot_shape_error_pct": 0.25, "ttft_mape_pct": 11.6, "ttft_shape_error_pct": 19.37}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.99, "tpot_shape_error_pct": 1.52, "ttft_mape_pct": 19.09, "ttft_shape_error_pct": 27.74}, "framework": "vllm", "id": "81741b1ab73abb68", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11200.488865439851, "interactivity_tok_s": 92.35006784849247, "output_per_gpu": 1461.3647847555105, "total_per_gpu": null, "tpot_error_pct": 7.62, "tpot_ms": 10.82836237479087, "tpot_relative": 1.076233, "ttft_error_pct": 19.79, "ttft_ms": 123.07415602878936, "ttft_relative": 1.197905, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10374.869337060867, "interactivity_tok_s": 89.423016031508, "output_per_gpu": 1379.6092803489498, "total_per_gpu": 2759.8405951519953, "tpot_error_pct": 11.15, "tpot_ms": 11.182803313720175, "tpot_relative": 1.111461, "ttft_error_pct": 28.95, "ttft_ms": 73.00079435312466, "ttft_relative": 0.710531, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 9370.34371291072, "interactivity_tok_s": 99.39020868252165, "output_per_gpu": 1525.2965242952582, "total_per_gpu": 3051.28077011018, "tpot_ms": 10.061353258591717, "tpot_relative": 1.0, "ttft_ms": 102.7411800939717, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13757.68477115139, "interactivity_tok_s": 75.08843803279689, "output_per_gpu": 2379.47013211441, "total_per_gpu": null, "tpot_error_pct": 7.36, "tpot_ms": 13.317629533900055, "tpot_relative": 1.323642, "ttft_error_pct": 3.41, "ttft_ms": 133.74975797163222, "ttft_relative": 1.301813, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12995.487225796207, "interactivity_tok_s": 71.44303940517736, "output_per_gpu": 2201.9933109800045, "total_per_gpu": 4408.824047111231, "tpot_error_pct": 12.84, "tpot_ms": 13.997164850849439, "tpot_relative": 1.391181, "ttft_error_pct": 9.23, "ttft_ms": 125.68624877421925, "ttft_relative": 1.223329, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 11542.568595353192, "interactivity_tok_s": 80.6129803495034, "output_per_gpu": 2475.3089462546036, "total_per_gpu": 4956.055748152979, "tpot_ms": 12.404950116773103, "tpot_relative": 1.232931, "ttft_ms": 138.47280156231818, "ttft_relative": 1.347783, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 119.59, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.96, "tpot_shape_error_pct": null, "ttft_mape_pct": 31.03, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "8d31e51ce31eba4c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7162.955031336496, "interactivity_tok_s": 144.84731385785486, "output_per_gpu": 285.6363038786589, "total_per_gpu": null, "tpot_error_pct": 14.08, "tpot_ms": 6.903821502560584, "tpot_relative": 1.140817, "ttft_error_pct": 119.59, "ttft_ms": 100.3456342170184, "ttft_relative": 2.195928, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6433.280364550008, "interactivity_tok_s": 145.0054955060949, "output_per_gpu": 282.68387177718654, "total_per_gpu": 563.2797333613629, "tpot_error_pct": 13.96, "tpot_ms": 6.896290354443622, "tpot_relative": 1.139573, "ttft_error_pct": 31.03, "ttft_ms": 31.51730113749978, "ttft_relative": 0.689713, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5662.881296599517, "interactivity_tok_s": 165.2442860610954, "output_per_gpu": 320.7964454541901, "total_per_gpu": 639.2233668043742, "tpot_ms": 6.051646467402039, "tpot_relative": 1.0, "ttft_ms": 45.69622308808903, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.0, "tpot_shape_error_pct": 0.78, "ttft_mape_pct": 97.61, "ttft_shape_error_pct": 1.46}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.28, "tpot_shape_error_pct": 0.57, "ttft_mape_pct": 18.58, "ttft_shape_error_pct": 6.9}, "framework": "vllm", "id": "1c19390481c831d6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8117.946754625291, "interactivity_tok_s": 127.76141594152999, "output_per_gpu": 504.0683467982267, "total_per_gpu": null, "tpot_error_pct": 14.45, "tpot_ms": 7.827089208667271, "tpot_relative": 1.144517, "ttft_error_pct": 96.18, "ttft_ms": 110.83449415867307, "ttft_relative": 1.961803, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7167.113389374967, "interactivity_tok_s": 128.7140881192752, "output_per_gpu": 498.3195373146598, "total_per_gpu": 1001.5124493847192, "tpot_error_pct": 13.6, "tpot_ms": 7.769157320784748, "tpot_relative": 1.136046, "ttft_error_pct": 21.3, "ttft_ms": 44.46317197500081, "ttft_relative": 0.787011, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 6326.430653898569, "interactivity_tok_s": 146.22511253891486, "output_per_gpu": 563.771340329175, "total_per_gpu": 1133.056149049703, "tpot_ms": 6.838770595809049, "tpot_relative": 1.0, "ttft_ms": 56.49624757352285, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10104.554695733063, "interactivity_tok_s": 102.47880222940701, "output_per_gpu": 809.9317828875654, "total_per_gpu": null, "tpot_error_pct": 13.56, "tpot_ms": 9.758115612645627, "tpot_relative": 1.426882, "ttft_error_pct": 99.03, "ttft_ms": 122.00242399658666, "ttft_relative": 2.159478, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9062.987999587498, "interactivity_tok_s": 103.02486385187862, "output_per_gpu": 805.1369650350174, "total_per_gpu": 1604.32688335954, "tpot_error_pct": 12.95, "tpot_ms": 9.706394773185282, "tpot_relative": 1.419319, "ttft_error_pct": 15.87, "ttft_ms": 51.57039314999982, "ttft_relative": 0.912811, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 8038.027447186869, "interactivity_tok_s": 116.37148461877902, "output_per_gpu": 906.9196947714001, "total_per_gpu": 1807.1405370224238, "tpot_ms": 8.593170425520452, "tpot_relative": 1.256537, "ttft_ms": 61.297000761624076, "ttft_relative": 1.084975, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 10.49, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.71, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.08, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "191545490e20f966", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 18622.149794968944, "interactivity_tok_s": 55.377382579303536, "output_per_gpu": 3515.813196695918, "total_per_gpu": null, "tpot_error_pct": 8.42, "tpot_ms": 18.057913780376737, "tpot_relative": 1.084213, "ttft_error_pct": 10.49, "ttft_ms": 148.90399764354095, "ttft_relative": 0.895073, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17579.334365288276, "interactivity_tok_s": 52.801747258181095, "output_per_gpu": 3261.3426119423902, "total_per_gpu": 6529.8498690719825, "tpot_error_pct": 13.71, "tpot_ms": 18.938767217500747, "tpot_relative": 1.1371, "ttft_error_pct": 0.08, "ttft_ms": 166.22701295546864, "ttft_relative": 0.999203, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 15475.432173801006, "interactivity_tok_s": 60.04088819005629, "output_per_gpu": 3695.5146934566046, "total_per_gpu": 7399.147838334337, "tpot_ms": 16.655316570843393, "tpot_relative": 1.0, "ttft_ms": 166.35967772931508, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 75.0, "points": 3, "status_counts": {"failed": 1, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.91, "tpot_shape_error_pct": 4.16, "ttft_mape_pct": 53.58, "ttft_shape_error_pct": 49.51}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.83, "tpot_shape_error_pct": 13.08, "ttft_mape_pct": 25.57, "ttft_shape_error_pct": 36.74}, "framework": "vllm", "id": "0b2afe03c8452615", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 22642.93839435831, "interactivity_tok_s": 45.501830099142495, "output_per_gpu": 5782.9950212038675, "total_per_gpu": null, "tpot_error_pct": 3.96, "tpot_ms": 21.97713801447396, "tpot_relative": 0.960429, "ttft_error_pct": 30.71, "ttft_ms": 160.32620555144683, "ttft_relative": 0.692949, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23228.075893814465, "interactivity_tok_s": 40.04694984905354, "output_per_gpu": 4951.246957307786, "total_per_gpu": 9897.708550490941, "tpot_error_pct": 9.13, "tpot_ms": 24.970690745967854, "tpot_relative": 1.091251, "ttft_error_pct": 10.71, "ttft_ms": 206.58518397421904, "ttft_relative": 0.892886, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 21318.019224552994, "interactivity_tok_s": 43.701261520912304, "output_per_gpu": 5374.191733672866, "total_per_gpu": 10743.189328466584, "tpot_ms": 22.882634624208077, "tpot_relative": 1.0, "ttft_ms": 231.36783113594674, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30608.385472746413, "interactivity_tok_s": 33.62286184091326, "output_per_gpu": 8556.086704840543, "total_per_gpu": null, "tpot_error_pct": 0.85, "tpot_ms": 29.74166817600194, "tpot_relative": 1.299748, "ttft_error_pct": 54.67, "ttft_ms": 182.65892869642784, "ttft_relative": 0.789474, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32054.191244785623, "interactivity_tok_s": 29.127510134575783, "output_per_gpu": 7200.970425582297, "total_per_gpu": 14395.307468069674, "tpot_error_pct": 14.45, "tpot_ms": 34.33180506606196, "tpot_relative": 1.500343, "ttft_error_pct": 1.93, "ttft_ms": 395.12958718828156, "ttft_relative": 1.707798, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 28055.023234952463, "interactivity_tok_s": 33.337263784349986, "output_per_gpu": 8194.524356628175, "total_per_gpu": 16381.500089095214, "tpot_ms": 29.996463011143856, "tpot_relative": 1.310883, "ttft_ms": 402.914745158688, "ttft_relative": 1.741447, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45577.479965062026, "interactivity_tok_s": 22.545574772353863, "output_per_gpu": 11491.991229034753, "total_per_gpu": null, "tpot_error_pct": 0.93, "tpot_ms": 44.354602182341935, "tpot_relative": 1.938352, "ttft_error_pct": 75.36, "ttft_ms": 202.7219325262289, "ttft_relative": 0.876189, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 47430.04911811876, "interactivity_tok_s": 19.789139461643973, "output_per_gpu": 9758.908549800557, "total_per_gpu": 19517.242354419686, "tpot_error_pct": 14.99, "tpot_ms": 50.532768336805965, "tpot_relative": 2.208346, "ttft_error_pct": 5.17, "ttft_ms": 865.1408255415915, "ttft_relative": 3.739244, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 41309.793678258684, "interactivity_tok_s": 22.75551268993614, "output_per_gpu": 11170.084777676671, "total_per_gpu": 22339.511699777653, "tpot_ms": 43.94539528183253, "tpot_relative": 1.920469, "ttft_ms": 822.6003706881329, "ttft_relative": 3.555379, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 77833.3954071708, "interactivity_tok_s": 12.543890890733717, "output_per_gpu": 11903.279313901592, "total_per_gpu": 23801.339553475602, "tpot_error_pct": 40.77, "tpot_ms": 79.72008117024589, "tpot_relative": 3.483868, "ttft_error_pct": 84.47, "ttft_ms": 4385.908614827301, "ttft_relative": 18.956432, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 80415.8793778945, "interactivity_tok_s": 17.657510876883556, "output_per_gpu": 11386.404412711992, "total_per_gpu": 22767.816378436335, "tpot_ms": 56.63312382886064, "tpot_relative": 2.474939, "ttft_ms": 28239.917176271018, "ttft_relative": 122.056368, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 46.67, "points": 14, "status_counts": {"failed": 16, "success": 14, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.23, "tpot_shape_error_pct": 3.84, "ttft_mape_pct": 59.57, "ttft_shape_error_pct": 39.11}, "aisimulate": {"coverage_pct": 100.0, "points": 30, "status_counts": {"failed": 0, "success": 30, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.91, "tpot_shape_error_pct": 3.73, "ttft_mape_pct": 32.12, "ttft_shape_error_pct": 15.1}, "gpu": "gb200", "precisions": ["fp4"], "rows": 30, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.81, "tpot_shape_error_pct": 1.92, "ttft_mape_pct": 25.6, "ttft_shape_error_pct": 17.54}, "framework": "dynamo-vllm", "id": "3a5c55d082247b7f", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7172.089270017184, "interactivity_tok_s": 130.66224206296698, "output_per_gpu": 608.4152729821864, "total_per_gpu": 1217.1048665764358, "tpot_error_pct": 0.66, "tpot_ms": 7.653320379410706, "tpot_relative": 0.993418, "ttft_error_pct": 34.29, "ttft_ms": 125.05756630156239, "ttft_relative": 0.657067, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 7281.344896496739, "interactivity_tok_s": 129.80218296677754, "output_per_gpu": 654.9487116640902, "total_per_gpu": 1212.164336000043, "tpot_ms": 7.704030680716263, "tpot_relative": 1.0, "ttft_ms": 190.3269950980757, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8298.004386176546, "interactivity_tok_s": 113.42718458807613, "output_per_gpu": 1052.1588783864563, "total_per_gpu": 2106.6291806069494, "tpot_error_pct": 2.68, "tpot_ms": 8.816228698892731, "tpot_relative": 1.144366, "ttft_error_pct": 35.04, "ttft_ms": 196.83424369453184, "ttft_relative": 1.03419, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 8193.083103360368, "interactivity_tok_s": 116.46799238083882, "output_per_gpu": 1162.2323970890834, "total_per_gpu": 2152.976143280627, "tpot_ms": 8.586049948642533, "tpot_relative": 1.114488, "ttft_ms": 303.01599157555756, "ttft_relative": 1.592081, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9694.763677265239, "interactivity_tok_s": 98.5458912420706, "output_per_gpu": 1802.0462316627543, "total_per_gpu": 3602.35079148763, "tpot_error_pct": 3.35, "tpot_ms": 10.147556507897168, "tpot_relative": 1.317175, "ttft_error_pct": 21.04, "ttft_ms": 350.1685769640637, "ttft_relative": 1.839826, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 9485.714390972771, "interactivity_tok_s": 101.85012027149705, "output_per_gpu": 1992.3379838648982, "total_per_gpu": 3683.664197833913, "tpot_ms": 9.818348739641616, "tpot_relative": 1.274443, "ttft_ms": 443.46867088552244, "ttft_relative": 2.330036, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11188.121407546021, "interactivity_tok_s": 87.96788550202618, "output_per_gpu": 3113.509581069721, "total_per_gpu": 6224.151062341731, "tpot_error_pct": 1.35, "tpot_ms": 11.367784894374514, "tpot_relative": 1.475563, "ttft_error_pct": 36.3, "ttft_ms": 715.7785718634773, "ttft_relative": 3.760783, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 11737.245808651778, "interactivity_tok_s": 86.77739419897411, "output_per_gpu": 3227.8965182000647, "total_per_gpu": 5969.922927708392, "tpot_ms": 11.52373851774201, "tpot_relative": 1.495806, "ttft_ms": 1123.6358405502415, "ttft_relative": 5.903712, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20566.56344615663, "interactivity_tok_s": 86.94052079129948, "output_per_gpu": 3374.901086566478, "total_per_gpu": 6749.603410317688, "tpot_error_pct": 1.24, "tpot_ms": 11.50211651481244, "tpot_relative": 1.493, "ttft_error_pct": 15.54, "ttft_ms": 9973.947873280576, "ttft_relative": 52.404273, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 22531.740809238636, "interactivity_tok_s": 85.86530186572013, "output_per_gpu": 3343.6889033128095, "total_per_gpu": 6185.070236130121, "tpot_ms": 11.646147841695628, "tpot_relative": 1.511695, "ttft_ms": 11808.638406404225, "ttft_relative": 62.043949, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 40087.303875268124, "interactivity_tok_s": 86.71844433217329, "output_per_gpu": 3455.1509727156663, "total_per_gpu": 6908.787010826836, "tpot_error_pct": 1.57, "tpot_ms": 11.531572178226812, "tpot_relative": 1.496823, "ttft_error_pct": 11.42, "ttft_ms": 29462.95701006208, "ttft_relative": 154.801777, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 44056.40757001934, "interactivity_tok_s": 85.35584205828079, "output_per_gpu": 3408.472769681251, "total_per_gpu": 6302.200650803252, "tpot_ms": 11.715659712163605, "tpot_relative": 1.520718, "ttft_ms": 33261.26386754383, "ttft_relative": 174.758519, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.58, "tpot_shape_error_pct": 3.38, "ttft_mape_pct": 36.51, "ttft_shape_error_pct": 30.24}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.91, "tpot_shape_error_pct": 2.96, "ttft_mape_pct": 41.74, "ttft_shape_error_pct": 9.41}, "framework": "dynamo-vllm", "id": "f43ec5e32a700cdc", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6300.925, "interactivity_tok_s": 164.23057973394646, "output_per_gpu": 67.20166666666667, "total_per_gpu": null, "tpot_error_pct": 9.62, "tpot_ms": 6.089, "tpot_relative": 1.096175, "ttft_error_pct": 16.25, "ttft_ms": 71.878, "ttft_relative": 0.837501, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5217.21126324999, "interactivity_tok_s": 177.29345734476883, "output_per_gpu": 75.63902132284822, "total_per_gpu": 152.01776338597546, "tpot_error_pct": 1.54, "tpot_ms": 5.640366063003541, "tpot_relative": 1.015409, "ttft_error_pct": 45.81, "ttft_ms": 46.50425709999971, "ttft_relative": 0.541853, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 5178.05071551702, "interactivity_tok_s": 180.02544495787907, "output_per_gpu": 85.7092898445174, "total_per_gpu": 153.6982545125882, "tpot_ms": 5.554770328349818, "tpot_relative": 1.0, "ttft_ms": 85.82442410988733, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6631.354, "interactivity_tok_s": 155.95757953836556, "output_per_gpu": 127.70422222222221, "total_per_gpu": null, "tpot_error_pct": 12.8, "tpot_ms": 6.412, "tpot_relative": 1.154323, "ttft_error_pct": 22.05, "ttft_ms": 71.878, "ttft_relative": 0.837501, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5558.8661576250115, "interactivity_tok_s": 168.5399375669569, "output_per_gpu": 145.01727905040286, "total_per_gpu": 288.9634055270271, "tpot_error_pct": 4.38, "tpot_ms": 5.933311797998761, "tpot_relative": 1.068147, "ttft_error_pct": 43.67, "ttft_ms": 51.94223547500003, "ttft_relative": 0.605215, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 5352.365572770941, "interactivity_tok_s": 175.9165993024973, "output_per_gpu": 166.5857279463146, "total_per_gpu": 296.3382272951807, "tpot_ms": 5.684511887820492, "tpot_relative": 1.023357, "ttft_ms": 92.2074421599973, "ttft_relative": 1.074373, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7346.431, "interactivity_tok_s": 140.62719729995783, "output_per_gpu": 230.28622222222222, "total_per_gpu": null, "tpot_error_pct": 15.09, "tpot_ms": 7.111, "tpot_relative": 1.280161, "ttft_error_pct": 32.83, "ttft_ms": 71.878, "ttft_relative": 0.837501, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6056.828589800008, "interactivity_tok_s": 152.6320270480199, "output_per_gpu": 262.1088801102153, "total_per_gpu": 527.007998259586, "tpot_error_pct": 6.04, "tpot_ms": 6.551704903227079, "tpot_relative": 1.179474, "ttft_error_pct": 41.78, "ttft_ms": 62.2948653250005, "ttft_relative": 0.725841, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 5756.053286901442, "interactivity_tok_s": 161.84887485576093, "output_per_gpu": 309.1300266438014, "total_per_gpu": 553.4323382574386, "tpot_ms": 6.17860334766736, "tpot_relative": 1.112306, "ttft_ms": 107.00710728560807, "ttft_relative": 1.246814, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8129.026, "interactivity_tok_s": 126.96800406297612, "output_per_gpu": 415.77133333333336, "total_per_gpu": null, "tpot_error_pct": 15.42, "tpot_ms": 7.876, "tpot_relative": 1.41788, "ttft_error_pct": 47.29, "ttft_ms": 71.878, "ttft_relative": 0.837501, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6765.817901575012, "interactivity_tok_s": 138.05163474561687, "output_per_gpu": 470.70485384582094, "total_per_gpu": 939.9207451455181, "tpot_error_pct": 6.16, "tpot_ms": 7.243666486403197, "tpot_relative": 1.304044, "ttft_error_pct": 39.68, "ttft_ms": 82.25934430312483, "ttft_relative": 0.958461, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 6439.9813331219775, "interactivity_tok_s": 146.54925113354247, "output_per_gpu": 560.3435003071311, "total_per_gpu": 995.6780710200024, "tpot_ms": 6.823644558161226, "tpot_relative": 1.22843, "ttft_ms": 136.3667148994864, "ttft_relative": 1.588903, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9076.324, "interactivity_tok_s": 113.61054305839582, "output_per_gpu": 744.0338888888889, "total_per_gpu": null, "tpot_error_pct": 9.97, "tpot_ms": 8.802, "tpot_relative": 1.584584, "ttft_error_pct": 64.13, "ttft_ms": 71.878, "ttft_relative": 0.837501, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7602.33795806563, "interactivity_tok_s": 123.13810124070945, "output_per_gpu": 830.3481148194745, "total_per_gpu": 1661.0706147229494, "tpot_error_pct": 1.46, "tpot_ms": 8.120963291818244, "tpot_relative": 1.46198, "ttft_error_pct": 37.74, "ttft_ms": 124.76347250312544, "ttft_relative": 1.453706, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 7567.9238604679995, "interactivity_tok_s": 124.93338652162825, "output_per_gpu": 945.9615144393811, "total_per_gpu": 1685.92116060412, "tpot_ms": 8.00426553575318, "tpot_relative": 1.440971, "ttft_ms": 200.4000419881777, "ttft_relative": 2.335, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.59, "tpot_shape_error_pct": 4.72, "ttft_mape_pct": 91.09, "ttft_shape_error_pct": 44.12}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.66, "tpot_shape_error_pct": 3.41, "ttft_mape_pct": 32.65, "ttft_shape_error_pct": 3.73}, "framework": "dynamo-vllm", "id": "1a3b9e25e1ea735a", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 18405.061, "interactivity_tok_s": 55.800457563752026, "output_per_gpu": 4783.539636363636, "total_per_gpu": null, "tpot_error_pct": 7.06, "tpot_ms": 17.921, "tpot_relative": 1.070643, "ttft_error_pct": 88.56, "ttft_ms": 71.878, "ttft_relative": 0.11438, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15482.029051321113, "interactivity_tok_s": 61.12273255330912, "output_per_gpu": 5333.050839134226, "total_per_gpu": 10665.787591374772, "tpot_error_pct": 2.26, "tpot_ms": 16.36052509805308, "tpot_relative": 0.977416, "ttft_error_pct": 33.88, "ttft_ms": 415.5134682481456, "ttft_relative": 0.661212, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 16039.946708869433, "interactivity_tok_s": 59.74235824818061, "output_per_gpu": 7072.217801863653, "total_per_gpu": 10307.03633439788, "tpot_ms": 16.73854245669075, "tpot_relative": 1.0, "ttft_ms": 628.4119059901286, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26304.667, "interactivity_tok_s": 38.99699723121319, "output_per_gpu": 6294.19290909091, "total_per_gpu": null, "tpot_error_pct": 12.12, "tpot_ms": 25.643, "tpot_relative": 1.531973, "ttft_error_pct": 93.61, "ttft_ms": 71.878, "ttft_relative": 0.11438, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22068.211360408593, "interactivity_tok_s": 43.26133490204711, "output_per_gpu": 7523.716437690484, "total_per_gpu": 15044.134050387247, "tpot_error_pct": 1.07, "tpot_ms": 23.1153292487209, "tpot_relative": 1.380964, "ttft_error_pct": 31.41, "ttft_ms": 771.2912577804266, "ttft_relative": 1.227366, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 22198.400469362376, "interactivity_tok_s": 43.72445427808173, "output_per_gpu": 10239.892120972321, "total_per_gpu": 14917.195683806507, "tpot_ms": 22.870496991000337, "tpot_relative": 1.366337, "ttft_ms": 1124.5171911324974, "ttft_relative": 1.789459, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 11}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.34, "tpot_shape_error_pct": 2.96, "ttft_mape_pct": 79.6, "ttft_shape_error_pct": 36.77}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.81, "tpot_shape_error_pct": 4.45, "ttft_mape_pct": 27.41, "ttft_shape_error_pct": 13.58}, "framework": "dynamo-vllm", "id": "7b720f36e3e16a60", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10846.114, "interactivity_tok_s": 94.94872768704899, "output_per_gpu": 1243.5455555555554, "total_per_gpu": null, "tpot_error_pct": 16.05, "tpot_ms": 10.532, "tpot_relative": 1.160533, "ttft_error_pct": 75.0, "ttft_ms": 71.878, "ttft_relative": 0.250007, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9134.441925402325, "interactivity_tok_s": 102.7970227400577, "output_per_gpu": 1379.899359579873, "total_per_gpu": 2762.8301361195086, "tpot_error_pct": 7.19, "tpot_ms": 9.727908195636122, "tpot_relative": 1.071929, "ttft_error_pct": 32.03, "ttft_ms": 195.4163095726565, "ttft_relative": 0.6797, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 8627.209674117876, "interactivity_tok_s": 110.19113054417777, "output_per_gpu": 1652.3854563108162, "total_per_gpu": 2947.590770369313, "tpot_ms": 9.075140576755228, "tpot_relative": 1.0, "ttft_ms": 287.50384248196497, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12272.176, "interactivity_tok_s": 83.85041086701325, "output_per_gpu": 2196.5118888888887, "total_per_gpu": null, "tpot_error_pct": 12.62, "tpot_ms": 11.926, "tpot_relative": 1.314139, "ttft_error_pct": 84.19, "ttft_ms": 71.878, "ttft_relative": 0.250007, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10338.811288944471, "interactivity_tok_s": 92.20004111987846, "output_per_gpu": 2441.762727681091, "total_per_gpu": 4881.165499605852, "tpot_error_pct": 2.42, "tpot_ms": 10.84598214766304, "tpot_relative": 1.195131, "ttft_error_pct": 22.8, "ttft_ms": 351.04523405585843, "ttft_relative": 1.221011, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 10207.434581247435, "interactivity_tok_s": 94.43162955718803, "output_per_gpu": 2778.9230021442772, "total_per_gpu": 4947.697232084809, "tpot_ms": 10.58967217540599, "tpot_relative": 1.166888, "ttft_ms": 454.7135433576841, "ttft_relative": 1.581591, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.15, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.69, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.85, "tpot_shape_error_pct": null, "ttft_mape_pct": 44.97, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "357f93f379af865f", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8129.026, "interactivity_tok_s": 126.96800406297612, "output_per_gpu": 374.1942, "total_per_gpu": null, "tpot_error_pct": 16.15, "tpot_ms": 7.876, "tpot_relative": 1.161547, "ttft_error_pct": 35.69, "ttft_ms": 71.878, "ttft_relative": 0.643139, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6690.421612812502, "interactivity_tok_s": 138.02564421952437, "output_per_gpu": 427.64350246342184, "total_per_gpu": 859.8394152353737, "tpot_error_pct": 6.85, "tpot_ms": 7.245030484404327, "tpot_relative": 1.068492, "ttft_error_pct": 44.97, "ttft_ms": 61.50207522499939, "ttft_relative": 0.550299, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 6310.246990782616, "interactivity_tok_s": 147.47933347746968, "output_per_gpu": 566.230981820452, "total_per_gpu": 912.3458037979742, "tpot_ms": 6.780611062042596, "tpot_relative": 1.0, "ttft_ms": 111.76127109210938, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.84, "tpot_shape_error_pct": 10.18, "ttft_mape_pct": 39.76, "ttft_shape_error_pct": 24.13}, "framework": "dynamo-vllm", "id": "1ea8eaba7d8dc315", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16964.006103567583, "interactivity_tok_s": 56.25586744925489, "output_per_gpu": 5359.6910884797335, "total_per_gpu": 10719.066521103963, "tpot_error_pct": 0.05, "tpot_ms": 17.775923567475715, "tpot_relative": 1.000537, "ttft_error_pct": 28.21, "ttft_ms": 594.1495409870079, "ttft_relative": 0.717866, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 17185.982542223042, "interactivity_tok_s": 56.28605276976758, "output_per_gpu": 6592.38634632552, "total_per_gpu": 10568.504004849252, "tpot_ms": 17.766390620610743, "tpot_relative": 1.0, "ttft_ms": 827.6609588564725, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 23068.420745301068, "interactivity_tok_s": 42.3814576309647, "output_per_gpu": 7903.75101647908, "total_per_gpu": 15804.036579200934, "tpot_error_pct": 6.63, "tpot_ms": 23.595224324455067, "tpot_relative": 1.328082, "ttft_error_pct": 61.93, "ttft_ms": 1329.0085884904886, "ttft_relative": 1.60574, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 23880.227040287868, "interactivity_tok_s": 45.193007680370286, "output_per_gpu": 9507.112687051798, "total_per_gpu": 15234.672839503828, "tpot_ms": 22.127316842298878, "tpot_relative": 1.245459, "ttft_ms": 3491.292206594744, "ttft_relative": 4.218264, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41329.82933489217, "interactivity_tok_s": 39.34880088248004, "output_per_gpu": 8781.208200623809, "total_per_gpu": 17568.797638919248, "tpot_error_pct": 13.84, "tpot_ms": 25.413735045868897, "tpot_relative": 1.430439, "ttft_error_pct": 29.14, "ttft_ms": 17939.105009732106, "ttft_relative": 21.674461, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 45865.82891774647, "interactivity_tok_s": 44.792710890964166, "output_per_gpu": 9836.420327183745, "total_per_gpu": 15773.311839412267, "tpot_ms": 22.325060933111008, "tpot_relative": 1.25659, "ttft_ms": 25317.49801334557, "ttft_relative": 30.589214, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.51, "tpot_shape_error_pct": 10.6, "ttft_mape_pct": 31.79, "ttft_shape_error_pct": 2.27}, "framework": "dynamo-vllm", "id": "029e8253b4c9bb72", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9566.196928238298, "interactivity_tok_s": 98.23880921930981, "output_per_gpu": 1700.1966583439369, "total_per_gpu": 3398.7500821322537, "tpot_error_pct": 3.96, "tpot_ms": 10.179276478887125, "tpot_relative": 1.039643, "ttft_error_pct": 32.56, "ttft_ms": 192.38973901992142, "ttft_relative": 0.674394, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 9302.53999319566, "interactivity_tok_s": 102.13330005183354, "output_per_gpu": 2029.6211901050808, "total_per_gpu": 3484.554996554227, "tpot_ms": 9.791125905972795, "tpot_relative": 1.0, "ttft_ms": 285.2781117185259, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13470.485025752601, "interactivity_tok_s": 71.5959413255272, "output_per_gpu": 4816.12248672011, "total_per_gpu": 9631.961330737935, "tpot_error_pct": 7.06, "tpot_ms": 13.967272187305602, "tpot_relative": 1.426524, "ttft_error_pct": 31.03, "ttft_ms": 607.964260812987, "ttft_relative": 2.131128, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 14718.477749243153, "interactivity_tok_s": 66.5422379928342, "output_per_gpu": 5151.65709548936, "total_per_gpu": 8848.731885279223, "tpot_ms": 15.028048802742223, "tpot_relative": 1.534864, "ttft_ms": 881.4743802767907, "ttft_relative": 3.089877, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.07, "tpot_shape_error_pct": 4.47, "ttft_mape_pct": 68.58, "ttft_shape_error_pct": 50.05}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.62, "tpot_shape_error_pct": 4.42, "ttft_mape_pct": 34.91, "ttft_shape_error_pct": 14.93}, "framework": "dynamo-vllm", "id": "a641c8c15b0d6992", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8902.414, "interactivity_tok_s": 115.84800741427247, "output_per_gpu": 682.8196, "total_per_gpu": null, "tpot_error_pct": 8.92, "tpot_ms": 8.632, "tpot_relative": 1.089179, "ttft_error_pct": 49.7, "ttft_ms": 71.878, "ttft_relative": 0.502993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7437.529975512507, "interactivity_tok_s": 125.46524421005654, "output_per_gpu": 771.2977858769701, "total_per_gpu": 1540.155755156138, "tpot_error_pct": 0.57, "tpot_ms": 7.970334783119531, "tpot_relative": 1.005691, "ttft_error_pct": 41.47, "ttft_ms": 83.64437415624981, "ttft_relative": 0.585333, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 7463.133529289189, "interactivity_tok_s": 126.1792294258171, "output_per_gpu": 968.5226180266495, "total_per_gpu": 1548.8768199768663, "tpot_ms": 7.925234640840131, "tpot_relative": 1.0, "ttft_ms": 142.90058458282147, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10846.114, "interactivity_tok_s": 94.94872768704899, "output_per_gpu": 1119.191, "total_per_gpu": null, "tpot_error_pct": 16.54, "tpot_ms": 10.532, "tpot_relative": 1.32892, "ttft_error_pct": 64.19, "ttft_ms": 71.878, "ttft_relative": 0.502993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9045.623623235972, "interactivity_tok_s": 103.19956682095405, "output_per_gpu": 1260.5034493728588, "total_per_gpu": 2521.575231088929, "tpot_error_pct": 7.22, "tpot_ms": 9.689963153963124, "tpot_relative": 1.222672, "ttft_error_pct": 38.46, "ttft_ms": 123.53420156875022, "ttft_relative": 0.864477, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 8518.771105301857, "interactivity_tok_s": 110.65485717613902, "output_per_gpu": 1681.7063829071942, "total_per_gpu": 2697.4690832870515, "tpot_ms": 9.03710894866741, "tpot_relative": 1.140295, "ttft_ms": 200.72685232007643, "ttft_relative": 1.404661, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12272.176, "interactivity_tok_s": 83.85041086701325, "output_per_gpu": 1976.8606, "total_per_gpu": null, "tpot_error_pct": 12.9, "tpot_ms": 11.926, "tpot_relative": 1.504813, "ttft_error_pct": 76.21, "ttft_ms": 71.878, "ttft_relative": 0.502993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10169.953962397649, "interactivity_tok_s": 92.13439799624358, "output_per_gpu": 2233.1459582031757, "total_per_gpu": 4471.197778913153, "tpot_error_pct": 2.75, "tpot_ms": 10.853709599760679, "tpot_relative": 1.369513, "ttft_error_pct": 35.0, "ttft_ms": 196.414758862499, "ttft_relative": 1.374485, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 10009.227529100826, "interactivity_tok_s": 94.66842270530564, "output_per_gpu": 2857.906361182483, "total_per_gpu": 4588.242133556404, "tpot_ms": 10.563184337748087, "tpot_relative": 1.332854, "ttft_ms": 302.1711922418035, "ttft_relative": 2.114555, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14298.739, "interactivity_tok_s": 71.90623427051125, "output_per_gpu": 3390.3576000000003, "total_per_gpu": null, "tpot_error_pct": 5.92, "tpot_ms": 13.907, "tpot_relative": 1.754775, "ttft_error_pct": 84.23, "ttft_ms": 71.878, "ttft_relative": 0.502993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11957.85547621208, "interactivity_tok_s": 79.28480184670379, "output_per_gpu": 3802.2338257074216, "total_per_gpu": 7600.792804755025, "tpot_error_pct": 3.94, "tpot_ms": 12.61275776325314, "tpot_relative": 1.591468, "ttft_error_pct": 24.72, "ttft_ms": 343.06385827734454, "ttft_relative": 2.400717, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 12547.67748833965, "interactivity_tok_s": 76.16221118736472, "output_per_gpu": 4537.589057892434, "total_per_gpu": 7271.002155969919, "tpot_ms": 13.129870895422474, "tpot_relative": 1.656717, "ttft_ms": 455.7129435694151, "ttft_relative": 3.189021, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.08, "tpot_shape_error_pct": 0.43, "ttft_mape_pct": 16.01, "ttft_shape_error_pct": 14.75}, "framework": "dynamo-vllm", "id": "9ee53f88eba90348", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 41396.47810314923, "interactivity_tok_s": 37.92147041255613, "output_per_gpu": 8770.776918366882, "total_per_gpu": 17537.708229532553, "tpot_error_pct": 8.77, "tpot_ms": 26.37028546416521, "tpot_relative": 1.087702, "ttft_error_pct": 23.53, "ttft_ms": 17100.093756243205, "ttft_relative": 0.764684, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 44701.443931933645, "interactivity_tok_s": 41.24726385810346, "output_per_gpu": 10132.881151150897, "total_per_gpu": 16237.435522312098, "tpot_ms": 24.244032366368454, "tpot_relative": 1.0, "ttft_ms": 22362.300119098654, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 80504.0951338288, "interactivity_tok_s": 37.662048774442354, "output_per_gpu": 8966.481176337766, "total_per_gpu": 17939.478226819178, "tpot_error_pct": 9.14, "tpot_ms": 26.551927803741915, "tpot_relative": 1.095194, "ttft_error_pct": 13.72, "ttft_ms": 56065.296454215364, "ttft_relative": 2.507135, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 87373.04459123507, "interactivity_tok_s": 41.10402624961839, "output_per_gpu": 10314.327671091109, "total_per_gpu": 16539.66599214847, "tpot_ms": 24.328516966371, "tpot_relative": 1.003485, "ttft_ms": 64980.824640385865, "ttft_relative": 2.90582, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 158559.68696924485, "interactivity_tok_s": 37.493379099660466, "output_per_gpu": 9080.432140204663, "total_per_gpu": 18162.767421260454, "tpot_error_pct": 9.34, "tpot_ms": 26.671375693876996, "tpot_relative": 1.100121, "ttft_error_pct": 10.78, "ttft_ms": 134006.3662283489, "ttft_relative": 5.992513, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 172646.58944428765, "interactivity_tok_s": 40.99701382319361, "output_per_gpu": 10413.100464159059, "total_per_gpu": 16694.64826888494, "tpot_ms": 24.392020460628302, "tpot_relative": 1.006104, "ttft_ms": 150193.60021206943, "ttft_relative": 6.716375, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.88, "tpot_shape_error_pct": 0.12, "ttft_mape_pct": 32.79, "ttft_shape_error_pct": 34.54}, "framework": "dynamo-vllm", "id": "4ad1f3848029503b", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42204.49723669268, "interactivity_tok_s": 28.132526323783292, "output_per_gpu": 10812.170917692058, "total_per_gpu": 21632.198958321616, "tpot_error_pct": 11.95, "tpot_ms": 35.54604334110577, "tpot_relative": 1.119484, "ttft_error_pct": 42.69, "ttft_ms": 9488.63130044491, "ttft_relative": 0.573102, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 45781.32398709008, "interactivity_tok_s": 31.493904941197666, "output_per_gpu": 13213.08443352364, "total_per_gpu": 19863.75259603617, "tpot_ms": 31.752175599281895, "tpot_relative": 1.0, "ttft_ms": 16556.624638556816, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 61803.635738980185, "interactivity_tok_s": 27.719428078483283, "output_per_gpu": 11031.476724184246, "total_per_gpu": 22065.97017410754, "tpot_error_pct": 11.81, "tpot_ms": 36.075780393760446, "tpot_relative": 1.136167, "ttft_error_pct": 22.89, "ttft_ms": 28593.888016118828, "ttft_relative": 1.727036, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 66785.49655339781, "interactivity_tok_s": 30.992924464617495, "output_per_gpu": 13542.322891581089, "total_per_gpu": 20353.724883576495, "tpot_ms": 32.26542887689194, "tpot_relative": 1.016164, "ttft_ms": 37083.744553606266, "ttft_relative": 2.239813, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 47.62, "points": 10, "status_counts": {"failed": 11, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.28, "tpot_shape_error_pct": 4.68, "ttft_mape_pct": 41.21, "ttft_shape_error_pct": 26.47}, "aisimulate": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.43, "tpot_shape_error_pct": 7.86, "ttft_mape_pct": 35.1, "ttft_shape_error_pct": 11.47}, "gpu": "gb300", "precisions": ["fp4"], "rows": 21, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.58, "tpot_shape_error_pct": 2.63, "ttft_mape_pct": 36.38, "ttft_shape_error_pct": 28.07}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.59, "tpot_shape_error_pct": 2.6, "ttft_mape_pct": 43.75, "ttft_shape_error_pct": 10.0}, "framework": "dynamo-vllm", "id": "0f178700f61cee7d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6483.108, "interactivity_tok_s": 159.5150741745095, "output_per_gpu": 65.31755555555554, "total_per_gpu": null, "tpot_error_pct": 15.58, "tpot_ms": 6.269, "tpot_relative": 1.155808, "ttft_error_pct": 19.42, "ttft_ms": 69.921, "ttft_relative": 0.80584, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5371.616408575071, "interactivity_tok_s": 172.1177110449045, "output_per_gpu": 73.40285074061006, "total_per_gpu": 147.52355332724912, "tpot_error_pct": 7.12, "tpot_ms": 5.809977334285522, "tpot_relative": 1.071178, "ttft_error_pct": 47.67, "ttft_ms": 45.40365457501032, "ttft_relative": 0.523277, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5059.234884123725, "interactivity_tok_s": 184.3687865551679, "output_per_gpu": 88.04601724020017, "total_per_gpu": 157.88859283693685, "tpot_ms": 5.423911599595923, "tpot_relative": 1.0, "ttft_ms": 86.76783862756565, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6832.974, "interactivity_tok_s": 151.26304643775526, "output_per_gpu": 123.72655555555555, "total_per_gpu": null, "tpot_error_pct": 18.55, "tpot_ms": 6.611, "tpot_relative": 1.218862, "ttft_error_pct": 25.51, "ttft_ms": 69.921, "ttft_relative": 0.80584, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5725.127486274976, "interactivity_tok_s": 163.5613312985187, "output_per_gpu": 140.79528860102215, "total_per_gpu": 280.55060984954366, "tpot_error_pct": 9.64, "tpot_ms": 6.113914530170228, "tpot_relative": 1.127215, "ttft_error_pct": 46.12, "ttft_ms": 50.57202100000002, "ttft_relative": 0.582843, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5254.124992315337, "interactivity_tok_s": 179.32694726920633, "output_per_gpu": 169.69609842760718, "total_per_gpu": 301.87124435504876, "tpot_ms": 5.576406754411517, "tpot_relative": 1.028115, "ttft_ms": 93.86131446663057, "ttft_relative": 1.081752, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7539.867, "interactivity_tok_s": 136.9487811558477, "output_per_gpu": 224.215, "total_per_gpu": null, "tpot_error_pct": 21.18, "tpot_ms": 7.302, "tpot_relative": 1.346261, "ttft_error_pct": 36.06, "ttft_ms": 69.921, "ttft_relative": 0.80584, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6222.229148637474, "interactivity_tok_s": 148.52619451219263, "output_per_gpu": 255.0611350989829, "total_per_gpu": 512.8374825981106, "tpot_error_pct": 11.73, "tpot_ms": 6.732819104968781, "tpot_relative": 1.241322, "ttft_error_pct": 43.33, "ttft_ms": 61.96579418125022, "ttft_relative": 0.714156, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5618.136017406505, "interactivity_tok_s": 165.953162258073, "output_per_gpu": 317.2653994464033, "total_per_gpu": 567.9970133283849, "tpot_ms": 6.025796594613271, "tpot_relative": 1.110969, "ttft_ms": 109.35229327114939, "ttft_relative": 1.260286, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9337.278, "interactivity_tok_s": 110.38745998454577, "output_per_gpu": 722.8893333333333, "total_per_gpu": null, "tpot_error_pct": 15.02, "tpot_ms": 9.059, "tpot_relative": 1.670197, "ttft_error_pct": 64.54, "ttft_ms": 69.921, "ttft_relative": 0.80584, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7801.25371781408, "interactivity_tok_s": 119.91312973575307, "output_per_gpu": 809.081714257031, "total_per_gpu": 1618.5282250616174, "tpot_error_pct": 5.88, "tpot_ms": 8.339370360890863, "tpot_relative": 1.53752, "ttft_error_pct": 37.86, "ttft_ms": 122.5421484000002, "ttft_relative": 1.412299, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 7446.556757848066, "interactivity_tok_s": 126.96810204694935, "output_per_gpu": 962.5868122902322, "total_per_gpu": 1715.5512681933353, "tpot_ms": 7.875993921924006, "tpot_relative": 1.452087, "ttft_ms": 197.20179243995517, "ttft_relative": 2.272752, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.08, "tpot_shape_error_pct": 17.13, "ttft_mape_pct": 34.94, "ttft_shape_error_pct": 20.83}, "framework": "dynamo-vllm", "id": "15324a4efbdd6f6c", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17153.655403982626, "interactivity_tok_s": 55.56178007256679, "output_per_gpu": 5311.629549199219, "total_per_gpu": 10622.946273099033, "tpot_error_pct": 4.18, "tpot_ms": 17.997983482421624, "tpot_relative": 1.041838, "ttft_error_pct": 24.44, "ttft_ms": 579.4240101820276, "ttft_relative": 0.755584, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 16672.62996702386, "interactivity_tok_s": 57.88638637183441, "output_per_gpu": 6798.463550388787, "total_per_gpu": 10898.874168555554, "tpot_ms": 17.27521897076938, "tpot_relative": 1.0, "ttft_ms": 766.855379993865, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24150.484553118422, "interactivity_tok_s": 40.235801946819926, "output_per_gpu": 7563.10426224697, "total_per_gpu": 15122.892429638818, "tpot_error_pct": 13.35, "tpot_ms": 24.853487481663976, "tpot_relative": 1.438679, "ttft_error_pct": 42.81, "ttft_ms": 1251.6361240347328, "ttft_relative": 1.632167, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 22392.285408305353, "interactivity_tok_s": 45.60691394002491, "output_per_gpu": 10153.975248188472, "total_per_gpu": 16271.2377583632, "tpot_ms": 21.92650003275915, "tpot_relative": 1.269246, "ttft_ms": 2188.688602725611, "ttft_relative": 2.854109, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 40751.7712061527, "interactivity_tok_s": 34.395140881022954, "output_per_gpu": 8939.93616509706, "total_per_gpu": 17886.368914278602, "tpot_error_pct": 30.72, "tpot_ms": 29.073874227150974, "tpot_relative": 1.682982, "ttft_error_pct": 37.55, "ttft_ms": 13992.637167100009, "ttft_relative": 18.246774, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 42877.928062250605, "interactivity_tok_s": 44.961664056340396, "output_per_gpu": 10532.492255388797, "total_per_gpu": 16889.506473337875, "tpot_ms": 22.241169693962476, "tpot_relative": 1.287461, "ttft_ms": 22406.957502251153, "ttft_relative": 29.219274, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.5, "tpot_shape_error_pct": 8.55, "ttft_mape_pct": 27.55, "ttft_shape_error_pct": 11.47}, "framework": "dynamo-vllm", "id": "5e1c89fe9dea309e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7247.890888720336, "interactivity_tok_s": 129.2211866734077, "output_per_gpu": 601.4400470113285, "total_per_gpu": 1203.1512696639365, "tpot_error_pct": 1.56, "tpot_ms": 7.738669066144624, "tpot_relative": 1.015553, "ttft_error_pct": 33.64, "ttft_ms": 122.20223316562502, "ttft_relative": 0.6636, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 7198.076017818676, "interactivity_tok_s": 131.23102164628565, "output_per_gpu": 662.4511188120301, "total_per_gpu": 1226.0496223086122, "tpot_ms": 7.620149469653267, "tpot_relative": 1.0, "ttft_ms": 184.15052716682112, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8512.259357984403, "interactivity_tok_s": 110.4489091391965, "output_per_gpu": 1025.4994374969133, "total_per_gpu": 2053.251732323945, "tpot_error_pct": 6.82, "tpot_ms": 9.053959951199884, "tpot_relative": 1.18816, "ttft_error_pct": 29.94, "ttft_ms": 192.6227587210943, "ttft_relative": 1.046007, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 8063.99924300149, "interactivity_tok_s": 117.97811056533878, "output_per_gpu": 1178.37544997509, "total_per_gpu": 2182.8803240884745, "tpot_ms": 8.476148627979414, "tpot_relative": 1.112334, "ttft_ms": 274.9327640176034, "ttft_relative": 1.492978, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10509.269065014076, "interactivity_tok_s": 90.5054665232689, "output_per_gpu": 1665.8414331849913, "total_per_gpu": 3330.0728360280455, "tpot_error_pct": 13.73, "tpot_ms": 11.049056354434686, "tpot_relative": 1.449979, "ttft_error_pct": 21.1, "ttft_ms": 334.50856648632873, "ttft_relative": 1.816495, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 9371.14800530369, "interactivity_tok_s": 102.93211886659974, "output_per_gpu": 2017.9272003051262, "total_per_gpu": 3730.9764918396895, "tpot_ms": 9.715140531557523, "tpot_relative": 1.274928, "ttft_ms": 423.94065999203576, "ttft_relative": 2.302142, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12105.994453515215, "interactivity_tok_s": 80.27645527927199, "output_per_gpu": 2881.1523994178406, "total_per_gpu": 5759.650741605799, "tpot_error_pct": 9.04, "tpot_ms": 12.456952621053361, "tpot_relative": 1.634739, "ttft_error_pct": 31.21, "ttft_ms": 630.4505559349575, "ttft_relative": 3.423561, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 11438.039657546426, "interactivity_tok_s": 87.53578083580933, "output_per_gpu": 3313.7677322498153, "total_per_gpu": 6128.739831129949, "tpot_ms": 11.423899923571799, "tpot_relative": 1.49917, "ttft_ms": 916.5114488199379, "ttft_relative": 4.976969, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20266.36236006154, "interactivity_tok_s": 75.88845438815939, "output_per_gpu": 3433.369889822332, "total_per_gpu": 6866.537573343603, "tpot_error_pct": 11.38, "tpot_ms": 13.177235036111455, "tpot_relative": 1.729262, "ttft_error_pct": 21.86, "ttft_ms": 8131.132864332901, "ttft_relative": 44.154817, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 21299.157527059284, "interactivity_tok_s": 84.52084467538485, "output_per_gpu": 3537.151532928968, "total_per_gpu": 6542.93246160715, "tpot_ms": 11.831400926489223, "tpot_relative": 1.552647, "ttft_ms": 10405.490827175901, "ttft_relative": 56.505355, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 13}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.68, "tpot_shape_error_pct": 3.38, "ttft_mape_pct": 24.98, "ttft_shape_error_pct": 10.68}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.58, "tpot_shape_error_pct": 3.05, "ttft_mape_pct": 47.86, "ttft_shape_error_pct": 11.73}, "framework": "dynamo-vllm", "id": "d2eab9f626aac969", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6483.108, "interactivity_tok_s": 159.5150741745095, "output_per_gpu": 58.785799999999995, "total_per_gpu": null, "tpot_error_pct": 17.04, "tpot_ms": 6.269, "tpot_relative": 1.170403, "ttft_error_pct": 19.82, "ttft_ms": 69.921, "ttft_relative": 0.801778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5465.415843600005, "interactivity_tok_s": 172.2673011047518, "output_per_gpu": 68.2292379946567, "total_per_gpu": 135.21098069959555, "tpot_error_pct": 8.38, "tpot_ms": 5.804932181481864, "tpot_relative": 1.083763, "ttft_error_pct": 51.64, "ttft_ms": 42.16941655000117, "ttft_relative": 0.483553, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5091.069788503228, "interactivity_tok_s": 186.69684337996736, "output_per_gpu": 91.57964142593384, "total_per_gpu": 145.73623301475098, "tpot_ms": 5.356276956246066, "tpot_relative": 1.0, "ttft_ms": 87.2074627521215, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6832.974, "interactivity_tok_s": 151.26304643775526, "output_per_gpu": 111.3538, "total_per_gpu": null, "tpot_error_pct": 20.2, "tpot_ms": 6.611, "tpot_relative": 1.234253, "ttft_error_pct": 18.99, "ttft_ms": 69.921, "ttft_relative": 0.801778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5644.972855299976, "interactivity_tok_s": 163.71452065921343, "output_per_gpu": 125.84795941477034, "total_per_gpu": 252.92666380842607, "tpot_error_pct": 11.06, "tpot_ms": 6.108193677466096, "tpot_relative": 1.14038, "ttft_error_pct": 47.37, "ttft_ms": 45.42709950000081, "ttft_relative": 0.520908, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5127.904307824792, "interactivity_tok_s": 181.8231730248489, "output_per_gpu": 172.90222930022475, "total_per_gpu": 279.0513584197171, "tpot_ms": 5.499849020142965, "tpot_relative": 1.026804, "ttft_ms": 86.31411984824808, "ttft_relative": 0.989756, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8291.772, "interactivity_tok_s": 124.42453651860146, "output_per_gpu": 366.6576, "total_per_gpu": null, "tpot_error_pct": 21.79, "tpot_ms": 8.037, "tpot_relative": 1.500483, "ttft_error_pct": 36.12, "ttft_ms": 69.921, "ttft_relative": 0.801778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6841.2070720437405, "interactivity_tok_s": 134.93824504199588, "output_per_gpu": 418.1502376143896, "total_per_gpu": 840.7518265091437, "tpot_error_pct": 12.3, "tpot_ms": 7.41079743321678, "tpot_relative": 1.383572, "ttft_error_pct": 44.58, "ttft_ms": 60.667739637500176, "ttft_relative": 0.695671, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 6142.244546556685, "interactivity_tok_s": 151.53372531179835, "output_per_gpu": 581.7767002813805, "total_per_gpu": 937.3940110847846, "tpot_ms": 6.599191024587979, "tpot_relative": 1.232048, "ttft_ms": 109.45995053225488, "ttft_relative": 1.255167, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.75, "tpot_shape_error_pct": 9.49, "ttft_mape_pct": 20.32, "ttft_shape_error_pct": 6.4}, "framework": "dynamo-vllm", "id": "5cbb59a644a5f791", "is_multinode": true, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 26727.983303747344, "interactivity_tok_s": 36.00508968191417, "output_per_gpu": 8529.846528536462, "total_per_gpu": 17055.953087451, "tpot_error_pct": 3.22, "tpot_ms": 27.77385110923118, "tpot_relative": 1.032152, "ttft_error_pct": 23.58, "ttft_ms": 1138.7948044400307, "ttft_relative": 0.764173, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 26284.50473322793, "interactivity_tok_s": 37.16271610879976, "output_per_gpu": 11548.150428519011, "total_per_gpu": 17348.7504307703, "tpot_ms": 26.9086898027674, "tpot_relative": 1.0, "ttft_ms": 1490.2317542723863, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 60564.39412301162, "interactivity_tok_s": 26.905442174450386, "output_per_gpu": 11263.868601509932, "total_per_gpu": 22530.81747986593, "tpot_error_pct": 12.89, "tpot_ms": 37.167201844004914, "tpot_relative": 1.381234, "ttft_error_pct": 21.43, "ttft_ms": 26349.945672750197, "ttft_relative": 17.681777, "unavailable_metrics": {}}, "concurrency": 6144, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 63845.719154238934, "interactivity_tok_s": 30.37322074005634, "output_per_gpu": 14197.00863767553, "total_per_gpu": 21337.698878871593, "tpot_ms": 32.92373925565278, "tpot_relative": 1.223536, "ttft_ms": 33537.935271108254, "ttft_relative": 22.505181, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 79673.93102837206, "interactivity_tok_s": 26.721217884911702, "output_per_gpu": 11383.221967032756, "total_per_gpu": 22768.82970980927, "tpot_error_pct": 13.14, "tpot_ms": 37.423443957794156, "tpot_relative": 1.390757, "ttft_error_pct": 15.95, "ttft_ms": 45222.79365347522, "ttft_relative": 30.346148, "unavailable_metrics": {}}, "concurrency": 8192, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 84255.88538403308, "interactivity_tok_s": 30.231857753008928, "output_per_gpu": 14309.217895216056, "total_per_gpu": 21507.22549437497, "tpot_ms": 33.07768937555522, "tpot_relative": 1.229257, "ttft_ms": 53807.680049320654, "ttft_relative": 36.106921, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.16, "tpot_shape_error_pct": 9.06, "ttft_mape_pct": 63.88, "ttft_shape_error_pct": 39.85}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.94, "tpot_shape_error_pct": 8.25, "ttft_mape_pct": 38.34, "ttft_shape_error_pct": 9.09}, "framework": "dynamo-vllm", "id": "93f41650f21bbd77", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 9233.955, "interactivity_tok_s": 111.63206072784104, "output_per_gpu": 657.9485999999999, "total_per_gpu": null, "tpot_error_pct": 14.25, "tpot_ms": 8.958, "tpot_relative": 1.142537, "ttft_error_pct": 50.81, "ttft_ms": 69.921, "ttft_relative": 0.491872, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7689.547270434357, "interactivity_tok_s": 121.29148824369764, "output_per_gpu": 746.2771239444189, "total_per_gpu": 1490.1935781878487, "tpot_error_pct": 5.15, "tpot_ms": 8.244601616156363, "tpot_relative": 1.051547, "ttft_error_pct": 41.87, "ttft_ms": 82.63607021249977, "ttft_relative": 0.581318, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 7384.082303444302, "interactivity_tok_s": 127.54374438688846, "output_per_gpu": 978.6691776774973, "total_per_gpu": 1565.1033600216733, "tpot_ms": 7.84044725052623, "tpot_relative": 1.0, "ttft_ms": 142.15296264610515, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11481.486, "interactivity_tok_s": 89.64589870013448, "output_per_gpu": 1056.6370000000002, "total_per_gpu": null, "tpot_error_pct": 24.56, "tpot_ms": 11.155, "tpot_relative": 1.42275, "ttft_error_pct": 65.24, "ttft_ms": 69.921, "ttft_relative": 0.491872, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9548.221748840613, "interactivity_tok_s": 97.67999573220995, "output_per_gpu": 1195.4739596174759, "total_per_gpu": 2391.486931259913, "tpot_error_pct": 14.32, "tpot_ms": 10.23751068480289, "tpot_relative": 1.30573, "ttft_error_pct": 39.34, "ttft_ms": 122.02752462499996, "ttft_relative": 0.858424, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 8444.089296733273, "interactivity_tok_s": 111.66352904276982, "output_per_gpu": 1696.5310622041065, "total_per_gpu": 2721.2479750600232, "tpot_ms": 8.955475512662472, "tpot_relative": 1.142215, "ttft_ms": 201.17859984293318, "ttft_relative": 1.415226, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13427.232, "interactivity_tok_s": 76.58727119552731, "output_per_gpu": 1805.5906, "total_per_gpu": null, "tpot_error_pct": 24.65, "tpot_ms": 13.057, "tpot_relative": 1.665339, "ttft_error_pct": 75.59, "ttft_ms": 69.921, "ttft_relative": 0.491872, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11099.764443029591, "interactivity_tok_s": 84.22256706024602, "output_per_gpu": 2050.4010406556213, "total_per_gpu": 4105.3064826255895, "tpot_error_pct": 13.35, "tpot_ms": 11.873302309637282, "tpot_relative": 1.514365, "ttft_error_pct": 33.82, "ttft_ms": 189.5557630390613, "ttft_relative": 1.333463, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 9912.142647554356, "interactivity_tok_s": 95.46918234029108, "output_per_gpu": 2885.8718510733092, "total_per_gpu": 4633.139489447918, "tpot_ms": 10.474584315969025, "tpot_relative": 1.335968, "ttft_ms": 286.4222366978993, "ttft_relative": 2.014888, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 160}, {"aic": {"coverage_pct": 77.0, "points": 77, "status_counts": {"failed": 23, "success": 77, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.6, "tpot_shape_error_pct": 7.17, "ttft_mape_pct": 45.45, "ttft_shape_error_pct": 38.67}, "aisimulate": {"coverage_pct": 100.0, "points": 100, "status_counts": {"failed": 0, "success": 100, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.19, "tpot_shape_error_pct": 8.02, "ttft_mape_pct": 34.1, "ttft_shape_error_pct": 23.12}, "gpu_skus": ["b200", "b300", "gb200", "gb300"], "gpus": [{"aic": {"coverage_pct": 66.67, "points": 22, "status_counts": {"failed": 11, "success": 22, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.99, "tpot_shape_error_pct": 8.01, "ttft_mape_pct": 40.63, "ttft_shape_error_pct": 31.5}, "aisimulate": {"coverage_pct": 100.0, "points": 33, "status_counts": {"failed": 0, "success": 33, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.91, "tpot_shape_error_pct": 11.03, "ttft_mape_pct": 34.02, "ttft_shape_error_pct": 34.4}, "gpu": "b200", "precisions": ["fp4"], "rows": 33, "topologies": [{"aic": {"coverage_pct": 60.0, "points": 3, "status_counts": {"failed": 2, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.24, "tpot_shape_error_pct": 28.14, "ttft_mape_pct": 76.41, "ttft_shape_error_pct": 66.85}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.27, "tpot_shape_error_pct": 7.71, "ttft_mape_pct": 29.81, "ttft_shape_error_pct": 12.59}, "framework": "dynamo-vllm", "id": "2883d39df7109e7a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11135.566, "interactivity_tok_s": 94.15309292910271, "output_per_gpu": 554.8851999999999, "total_per_gpu": null, "tpot_error_pct": 18.09, "tpot_ms": 10.621, "tpot_relative": 1.180915, "ttft_error_pct": 57.45, "ttft_ms": 270.283, "ttft_relative": 0.425528, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9208.117075074988, "interactivity_tok_s": 105.14153163534397, "output_per_gpu": 621.9136208026723, "total_per_gpu": 5561.349618858932, "tpot_error_pct": 5.75, "tpot_ms": 9.510989467684755, "tpot_relative": 1.057496, "ttft_error_pct": 34.29, "ttft_ms": 417.3570954999994, "ttft_relative": 0.657079, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 8948.99796587415, "interactivity_tok_s": 111.18680004137117, "output_per_gpu": 799.4022495681761, "total_per_gpu": 5723.6176412423365, "tpot_ms": 8.993873370111496, "tpot_relative": 1.0, "ttft_ms": 635.1704237546073, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14431.672, "interactivity_tok_s": 72.23867658744491, "output_per_gpu": 851.452, "total_per_gpu": null, "tpot_error_pct": 25.17, "tpot_ms": 13.843, "tpot_relative": 1.539159, "ttft_error_pct": 73.12, "ttft_ms": 270.283, "ttft_relative": 0.425528, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11942.956705204844, "interactivity_tok_s": 81.20587535613036, "output_per_gpu": 957.0081308694498, "total_per_gpu": 8626.258812806507, "tpot_error_pct": 11.35, "tpot_ms": 12.314379909267346, "tpot_relative": 1.369196, "ttft_error_pct": 39.65, "ttft_ms": 606.9552296687528, "ttft_relative": 0.955579, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 11185.21726435647, "interactivity_tok_s": 90.42172166334608, "output_per_gpu": 1273.559465132294, "total_per_gpu": 9191.065916659189, "tpot_ms": 11.059289533583017, "tpot_relative": 1.229647, "ttft_ms": 1005.6509291644034, "ttft_relative": 1.583277, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25612.039, "interactivity_tok_s": 40.368157597287265, "output_per_gpu": 1227.5711999999999, "total_per_gpu": null, "tpot_error_pct": 77.47, "tpot_ms": 24.772, "tpot_relative": 2.754319, "ttft_error_pct": 98.67, "ttft_ms": 270.283, "ttft_relative": 0.425528, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28395.0033039137, "interactivity_tok_s": 61.48790570900782, "output_per_gpu": 1598.9266493152668, "total_per_gpu": 14386.202400797198, "tpot_error_pct": 16.51, "tpot_ms": 16.263360875104624, "tpot_relative": 1.808271, "ttft_error_pct": 33.83, "ttft_ms": 13415.97517647958, "ttft_relative": 21.121851, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 33130.22569598343, "interactivity_tok_s": 71.64156449098212, "output_per_gpu": 1708.973459751624, "total_per_gpu": 12306.164758459685, "tpot_ms": 13.958377474097107, "tpot_relative": 1.551987, "ttft_ms": 20274.15471097811, "ttft_relative": 31.919236, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 55400.691686734346, "interactivity_tok_s": 61.17728374751702, "output_per_gpu": 1629.4925896595782, "total_per_gpu": 14665.551688077065, "tpot_error_pct": 15.11, "tpot_ms": 16.34593657552811, "tpot_relative": 1.817452, "ttft_error_pct": 22.66, "ttft_ms": 40344.390123915844, "ttft_relative": 63.517426, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 65244.22309116808, "interactivity_tok_s": 70.41909327102992, "output_per_gpu": 1728.2060153502027, "total_per_gpu": 12450.288774170771, "tpot_ms": 14.200694066752423, "tpot_relative": 1.57893, "ttft_ms": 52165.756337139006, "ttft_relative": 82.128755, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 109274.14717270862, "interactivity_tok_s": 60.97822335705782, "output_per_gpu": 1647.6466141145047, "total_per_gpu": 14825.778360988767, "tpot_error_pct": 12.63, "tpot_ms": 16.3992970760152, "tpot_relative": 1.823385, "ttft_error_pct": 18.65, "ttft_ms": 94173.47396859931, "ttft_relative": 148.264892, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 129164.23807377313, "interactivity_tok_s": 68.67804725164642, "output_per_gpu": 1737.0234835265837, "total_per_gpu": 12509.381405810189, "tpot_ms": 14.5606935551888, "tpot_relative": 1.618957, "ttft_ms": 115757.51921050723, "ttft_relative": 182.246394, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.96, "tpot_shape_error_pct": 2.66, "ttft_mape_pct": 26.73, "ttft_shape_error_pct": 25.64}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.14, "tpot_shape_error_pct": 2.03, "ttft_mape_pct": 43.65, "ttft_shape_error_pct": 11.87}, "framework": "dynamo-vllm", "id": "5d3761d497c2e0d7", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7249.189, "interactivity_tok_s": 146.5845793022574, "output_per_gpu": 107.9624, "total_per_gpu": null, "tpot_error_pct": 12.96, "tpot_ms": 6.822, "tpot_relative": 1.129565, "ttft_error_pct": 11.62, "ttft_ms": 270.283, "ttft_relative": 0.883752, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5918.256455774986, "interactivity_tok_s": 159.6106430216488, "output_per_gpu": 121.07484317329127, "total_per_gpu": 1088.3906327428144, "tpot_error_pct": 3.74, "tpot_ms": 6.2652463586927905, "tpot_relative": 1.03738, "ttft_error_pct": 47.78, "ttft_ms": 159.70907762500224, "ttft_relative": 0.522205, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 5856.913179738331, "interactivity_tok_s": 165.57682282284478, "output_per_gpu": 152.5356397982608, "total_per_gpu": 1098.0245186292955, "tpot_ms": 6.03949262313076, "tpot_relative": 1.0, "ttft_ms": 305.83595147472806, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8163.751, "interactivity_tok_s": 129.60082944530845, "output_per_gpu": 191.0538, "total_per_gpu": null, "tpot_error_pct": 16.61, "tpot_ms": 7.716, "tpot_relative": 1.277591, "ttft_error_pct": 27.19, "ttft_ms": 270.283, "ttft_relative": 0.883752, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6756.383706912515, "interactivity_tok_s": 141.46146931088663, "output_per_gpu": 215.02695044793285, "total_per_gpu": 1912.7641396162107, "tpot_error_pct": 6.83, "tpot_ms": 7.069062726913453, "tpot_relative": 1.170473, "ttft_error_pct": 41.31, "ttft_ms": 217.87969637500024, "ttft_relative": 0.712407, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 6491.126841549703, "interactivity_tok_s": 151.1302482310342, "output_per_gpu": 279.35385732001026, "total_per_gpu": 1989.9163998107554, "tpot_ms": 6.616809088219659, "tpot_relative": 1.09559, "ttft_ms": 371.23970252578147, "ttft_relative": 1.213852, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8913.61, "interactivity_tok_s": 118.35720203574388, "output_per_gpu": 348.758, "total_per_gpu": null, "tpot_error_pct": 15.31, "tpot_ms": 8.449, "tpot_relative": 1.398959, "ttft_error_pct": 41.37, "ttft_ms": 270.283, "ttft_relative": 0.883752, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7291.382210037486, "interactivity_tok_s": 130.16354238474568, "output_per_gpu": 386.97905711710166, "total_per_gpu": 3492.3891307544495, "tpot_error_pct": 4.85, "tpot_ms": 7.682642786750044, "tpot_relative": 1.272068, "ttft_error_pct": 41.85, "ttft_ms": 268.1004654124994, "ttft_relative": 0.876615, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 7155.200984775001, "interactivity_tok_s": 136.4762876004596, "output_per_gpu": 494.61528214554505, "total_per_gpu": 3573.3715624736733, "tpot_ms": 7.327280200700831, "tpot_relative": 1.213228, "ttft_ms": 461.03330830519553, "ttft_relative": 1.507453, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 25.0, "points": 1, "status_counts": {"failed": 3, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.76, "tpot_shape_error_pct": null, "ttft_mape_pct": 82.25, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.06, "tpot_shape_error_pct": 51.61, "ttft_mape_pct": 48.71, "ttft_shape_error_pct": 25.42}, "framework": "dynamo-vllm", "id": "e0e4f1b7c546843b", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 24488.785, "interactivity_tok_s": 42.24043254202923, "output_per_gpu": 1659.631, "total_per_gpu": null, "tpot_error_pct": 9.76, "tpot_ms": 23.674, "tpot_relative": 1.097553, "ttft_error_pct": 82.25, "ttft_ms": 270.283, "ttft_relative": 0.177487, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20227.228256656712, "interactivity_tok_s": 47.788161596026555, "output_per_gpu": 1877.78270830577, "total_per_gpu": 16895.185353232202, "tpot_error_pct": 2.99, "tpot_ms": 20.925684659171885, "tpot_relative": 0.970138, "ttft_error_pct": 37.32, "ttft_ms": 954.4922018187506, "ttft_relative": 0.626786, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 21388.42217717629, "interactivity_tok_s": 46.36109680031696, "output_per_gpu": 2660.652664402142, "total_per_gpu": 15965.934533357316, "tpot_ms": 21.569808935002662, "tpot_relative": 1.0, "ttft_ms": 1522.8367969348255, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 30716.354095215676, "interactivity_tok_s": 32.715078782161, "output_per_gpu": 2486.2961899332972, "total_per_gpu": 22376.8463365356, "tpot_error_pct": 14.7, "tpot_ms": 30.566944578023868, "tpot_relative": 1.417117, "ttft_error_pct": 74.72, "ttft_ms": 2561.855271150582, "ttft_relative": 1.682291, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 34680.042423825944, "interactivity_tok_s": 37.52286775371231, "output_per_gpu": 3269.333080062151, "total_per_gpu": 19627.357591331824, "tpot_ms": 26.6504150632534, "tpot_relative": 1.235542, "ttft_ms": 10134.291229244081, "ttft_relative": 6.654877, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 58282.131310727054, "interactivity_tok_s": 25.723263643681385, "output_per_gpu": 2598.2733893080676, "total_per_gpu": 23379.664705492123, "tpot_error_pct": 43.48, "tpot_ms": 38.87531589505899, "tpot_relative": 1.802302, "ttft_error_pct": 46.6, "ttft_ms": 22518.15460686187, "ttft_relative": 14.786978, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 67115.07800435061, "interactivity_tok_s": 36.90755354169556, "output_per_gpu": 3354.2901056690234, "total_per_gpu": 20130.266264621772, "tpot_ms": 27.09472463056296, "tpot_relative": 1.256141, "ttft_ms": 42166.894811534665, "ttft_relative": 27.6897, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 114082.95418556621, "interactivity_tok_s": 20.092476084112967, "output_per_gpu": 2642.226835960287, "total_per_gpu": 23758.45942490577, "tpot_error_pct": 83.08, "tpot_ms": 49.76987384798709, "tpot_relative": 2.307386, "ttft_error_pct": 36.2, "ttft_ms": 68297.94883053399, "ttft_relative": 44.849158, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26842607912", "measured": {"e2e_ms": 132107.53747871894, "interactivity_tok_s": 36.78492471311605, "output_per_gpu": 3393.565421596354, "total_per_gpu": 20353.422935232284, "tpot_ms": 27.18504952229628, "tpot_relative": 1.260329, "ttft_ms": 107057.30097271346, "ttft_relative": 70.301231, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.72, "tpot_shape_error_pct": 3.14, "ttft_mape_pct": 50.56, "ttft_shape_error_pct": 11.91}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.73, "tpot_shape_error_pct": 3.29, "ttft_mape_pct": 13.07, "ttft_shape_error_pct": 1.56}, "framework": "trt", "id": "386e59b3d398583d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8808.558051007092, "interactivity_tok_s": 121.2977952854896, "output_per_gpu": 456.4038812150566, "total_per_gpu": null, "tpot_error_pct": 2.32, "tpot_ms": 8.244172927021257, "tpot_relative": 1.023185, "ttft_error_pct": 65.32, "ttft_ms": 374.76914666434703, "ttft_relative": 1.6532, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7971.167802074982, "interactivity_tok_s": 118.18953729257251, "output_per_gpu": 451.5363987361019, "total_per_gpu": 4059.0429344880586, "tpot_error_pct": 5.01, "tpot_ms": 8.46098582757413, "tpot_relative": 1.050094, "ttft_error_pct": 14.08, "ttft_ms": 194.78012332500032, "ttft_relative": 0.859224, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 7631.6314342502665, "interactivity_tok_s": 124.11010285932974, "output_per_gpu": 471.68012522286233, "total_per_gpu": 4240.123022160307, "tpot_ms": 8.057361785715633, "tpot_relative": 1.0, "ttft_ms": 226.69318177486275, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11971.234480375857, "interactivity_tok_s": 88.51087064379958, "output_per_gpu": 677.123819103743, "total_per_gpu": null, "tpot_error_pct": 7.49, "tpot_ms": 11.29804726500058, "tpot_relative": 1.402202, "ttft_error_pct": 58.87, "ttft_ms": 413.3321282802621, "ttft_relative": 1.823311, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10736.237332099958, "interactivity_tok_s": 88.1273690810777, "output_per_gpu": 680.0373268664805, "total_per_gpu": 6049.246430370775, "tpot_error_pct": 7.96, "tpot_ms": 11.34721268122726, "tpot_relative": 1.408304, "ttft_error_pct": 12.01, "ttft_ms": 228.92760256249932, "ttft_relative": 1.009857, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 9991.439996799909, "interactivity_tok_s": 95.14348377069302, "output_per_gpu": 730.3037225686909, "total_per_gpu": 6496.389260265616, "tpot_ms": 10.510441286868552, "tpot_relative": 1.304452, "ttft_ms": 260.17300938749486, "ttft_relative": 1.147688, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16525.19033556752, "interactivity_tok_s": 63.79841133133915, "output_per_gpu": 988.4894073038655, "total_per_gpu": null, "tpot_error_pct": 4.76, "tpot_ms": 15.674371495027785, "tpot_relative": 1.945348, "ttft_error_pct": 42.47, "ttft_ms": 490.3082961540955, "ttft_relative": 2.162872, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14572.258846225019, "interactivity_tok_s": 64.09226841120254, "output_per_gpu": 980.7673473125677, "total_per_gpu": 8851.179826294314, "tpot_error_pct": 4.28, "tpot_ms": 15.602505961939277, "tpot_relative": 1.936429, "ttft_error_pct": 13.55, "ttft_ms": 297.53270648125016, "ttft_relative": 1.312491, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 14035.386504599683, "interactivity_tok_s": 66.83279795874722, "output_per_gpu": 1018.5130802829453, "total_per_gpu": 9191.825618705287, "tpot_ms": 14.9627133764062, "tpot_relative": 1.857024, "ttft_ms": 344.1529355685816, "ttft_relative": 1.518144, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23912.895524433985, "interactivity_tok_s": 43.97458766821148, "output_per_gpu": 1376.4625734558915, "total_per_gpu": null, "tpot_error_pct": 0.3, "tpot_ms": 22.74040651716864, "tpot_relative": 2.822314, "ttft_error_pct": 35.57, "ttft_ms": 649.4596573704646, "ttft_relative": 2.864928, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21047.794293171974, "interactivity_tok_s": 44.86006036111528, "output_per_gpu": 1378.5559490329922, "total_per_gpu": 12327.486238097525, "tpot_error_pct": 1.68, "tpot_ms": 22.291543790850547, "tpot_relative": 2.766606, "ttft_error_pct": 12.66, "ttft_ms": 418.4119174749996, "ttft_relative": 1.845719, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 21458.7546510501, "interactivity_tok_s": 44.10768547083289, "output_per_gpu": 1351.7331591897005, "total_per_gpu": 12087.628310755144, "tpot_ms": 22.671785865102137, "tpot_relative": 2.813798, "ttft_ms": 479.0752512625602, "ttft_relative": 2.11332, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.28, "ttft_shape_error_pct": null}, "framework": "trt", "id": "27934046fc403cb7", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 76365.03374609973, "interactivity_tok_s": 15.983423482331858, "output_per_gpu": 1517.2304333243924, "total_per_gpu": 13651.147857096097, "tpot_error_pct": 36.08, "tpot_ms": 62.56481917690563, "tpot_relative": 1.360751, "ttft_error_pct": 30.28, "ttft_ms": 18621.702693704363, "ttft_relative": 1.302845, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 56658.32758219064, "interactivity_tok_s": 21.749466696144133, "output_per_gpu": 2035.5683957879767, "total_per_gpu": 18314.848248362538, "tpot_ms": 45.97813886522954, "tpot_relative": 1.0, "ttft_ms": 14293.105692189845, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.11, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.59, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.93, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.47, "ttft_shape_error_pct": null}, "framework": "trt", "id": "c6468189f49096bb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6680.540786216693, "interactivity_tok_s": 158.04193643837382, "output_per_gpu": 151.24009451295112, "total_per_gpu": null, "tpot_error_pct": 6.11, "tpot_ms": 6.327434493248794, "tpot_relative": 0.938892, "ttft_error_pct": 30.59, "ttft_ms": 207.57529962317693, "ttft_relative": 1.305918, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6061.132130700004, "interactivity_tok_s": 154.45433073919395, "output_per_gpu": 148.08340157602285, "total_per_gpu": 1331.1814652475218, "tpot_error_pct": 3.93, "tpot_ms": 6.474405704353892, "tpot_relative": 0.9607, "ttft_error_pct": 30.47, "ttft_ms": 110.51513505000007, "ttft_relative": 0.695283, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 6353.5030668499985, "interactivity_tok_s": 148.3843439722796, "output_per_gpu": 141.49936936538754, "total_per_gpu": 1271.9949423009182, "tpot_ms": 6.739255458020658, "tpot_relative": 1.0, "ttft_ms": 158.9497644498806, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 33.33, "points": 1, "status_counts": {"failed": 2, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 48.64, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 128.08, "tpot_shape_error_pct": 5.13, "ttft_mape_pct": 58.23, "ttft_shape_error_pct": 215.14}, "framework": "trt", "id": "0876fc5093ba0e74", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 45170.0880210977, "interactivity_tok_s": 23.209520275575503, "output_per_gpu": 1461.0835345822868, "total_per_gpu": null, "tpot_error_pct": 30.39, "tpot_ms": 43.085767742142785, "tpot_relative": 1.303884, "ttft_error_pct": 48.64, "ttft_ms": 1093.3476208856318, "ttft_relative": 0.513567, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 68345.99733518316, "interactivity_tok_s": 13.72235871257219, "output_per_gpu": 856.0192973759646, "total_per_gpu": 7701.958608490652, "tpot_error_pct": 120.53, "tpot_ms": 72.87376907614411, "tpot_relative": 2.205343, "ttft_error_pct": 51.2, "ttft_ms": 1038.9284287433609, "ttft_relative": 0.488005, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 32582.52965927499, "interactivity_tok_s": 30.262510629886417, "output_per_gpu": 1775.7086593983972, "total_per_gpu": 15976.783043733389, "tpot_ms": 33.04418500599972, "tpot_relative": 1.0, "ttft_ms": 2128.929119654269, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 113843.86328229184, "interactivity_tok_s": 8.49303576827838, "output_per_gpu": 1024.899814644324, "total_per_gpu": 9224.17278982965, "tpot_error_pct": 120.71, "tpot_ms": 117.74352861376322, "tpot_relative": 3.563215, "ttft_error_pct": 7.95, "ttft_ms": 5086.801510892974, "ttft_relative": 2.389371, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 54665.85086799532, "interactivity_tok_s": 18.744767361459374, "output_per_gpu": 2114.904002688343, "total_per_gpu": 19034.289670029524, "tpot_ms": 53.34822143784371, "tpot_relative": 1.614451, "ttft_ms": 5526.293098364841, "ttft_relative": 2.595809, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 246849.38454648, "interactivity_tok_s": 7.616286596688724, "output_per_gpu": 934.699803059617, "total_per_gpu": 8410.57299272226, "tpot_error_pct": 143.0, "tpot_ms": 131.29758016652926, "tpot_relative": 3.973394, "ttft_error_pct": 115.54, "ttft_ms": 125728.96551512279, "ttft_relative": 59.057375, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 108099.67167820154, "interactivity_tok_s": 18.507322492406455, "output_per_gpu": 2119.48143513386, "total_per_gpu": 19071.420854654956, "tpot_ms": 54.03266736235344, "tpot_relative": 1.635164, "ttft_ms": 58333.125580179476, "ttft_relative": 27.40022, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.19, "tpot_shape_error_pct": null, "ttft_mape_pct": 39.5, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.92, "tpot_shape_error_pct": null, "ttft_mape_pct": 25.96, "ttft_shape_error_pct": null}, "framework": "trt", "id": "2932672a7b71308d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6162.596308934477, "interactivity_tok_s": 171.10763525585986, "output_per_gpu": 82.02200615286819, "total_per_gpu": null, "tpot_error_pct": 3.19, "tpot_ms": 5.844274561475206, "tpot_relative": 0.968062, "ttft_error_pct": 39.5, "ttft_ms": 183.90343254534022, "ttft_relative": 1.394967, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5595.4555364, "interactivity_tok_s": 167.1752030805751, "output_per_gpu": 80.1318136166312, "total_per_gpu": 720.3372148928275, "tpot_error_pct": 0.92, "tpot_ms": 5.981748378783306, "tpot_relative": 0.990834, "ttft_error_pct": 25.96, "ttft_ms": 97.60687272499933, "ttft_relative": 0.74038, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27455221794", "measured": {"e2e_ms": 5680.7938623251175, "interactivity_tok_s": 165.64280516269997, "output_per_gpu": 79.01784962422938, "total_per_gpu": 710.3233429540664, "tpot_ms": 6.037086844899578, "tpot_relative": 1.0, "ttft_ms": 131.83356732461107, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.05, "tpot_shape_error_pct": 2.68, "ttft_mape_pct": 5.45, "ttft_shape_error_pct": 9.91}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.29, "tpot_shape_error_pct": 2.84, "ttft_mape_pct": 42.36, "ttft_shape_error_pct": 3.83}, "framework": "vllm", "id": "34b4829696141f66", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6897.881998761866, "interactivity_tok_s": 151.75796421105198, "output_per_gpu": 148.30639320643976, "total_per_gpu": null, "tpot_error_pct": 11.56, "tpot_ms": 6.589440002036964, "tpot_relative": 1.115578, "ttft_error_pct": 10.01, "ttft_ms": 156.88487667805222, "ttft_relative": 1.100066, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6040.814317449981, "interactivity_tok_s": 154.28993877404983, "output_per_gpu": 148.50814613943217, "total_per_gpu": 1334.9996655607072, "tpot_error_pct": 9.73, "tpot_ms": 6.481304017266166, "tpot_relative": 1.097271, "ttft_error_pct": 41.24, "ttft_ms": 83.80551057500075, "ttft_relative": 0.587639, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5571.307471475666, "interactivity_tok_s": 169.29790662601488, "output_per_gpu": 160.58586828452698, "total_per_gpu": 1443.5711846563556, "tpot_ms": 5.906747578450783, "tpot_relative": 1.0, "ttft_ms": 142.61403177551983, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8033.103236971484, "interactivity_tok_s": 130.14787598057634, "output_per_gpu": 254.6960918644127, "total_per_gpu": null, "tpot_error_pct": 14.55, "tpot_ms": 7.683567576233384, "tpot_relative": 1.300812, "ttft_error_pct": 0.89, "ttft_ms": 172.81360648473247, "ttft_relative": 1.211757, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7104.1985496000025, "interactivity_tok_s": 132.11219554100717, "output_per_gpu": 256.1428976587616, "total_per_gpu": 2278.5094995694494, "tpot_error_pct": 12.85, "tpot_ms": 7.569323906131009, "tpot_relative": 1.281471, "ttft_error_pct": 43.49, "ttft_ms": 98.53506647499971, "ttft_relative": 0.690921, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 6381.973580713566, "interactivity_tok_s": 149.08570937235666, "output_per_gpu": 284.15701020864947, "total_per_gpu": 2527.707982722253, "tpot_ms": 6.707551006799711, "tpot_relative": 1.135574, "ttft_ms": 174.36570563804707, "ttft_relative": 1.222641, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.68, "tpot_shape_error_pct": null, "ttft_mape_pct": 0.88, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.35, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.16, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "78999e67df5b6085", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6452.644612763267, "interactivity_tok_s": 162.15171559133069, "output_per_gpu": 79.26982356788379, "total_per_gpu": null, "tpot_error_pct": 13.68, "tpot_ms": 6.167063952134122, "tpot_relative": 1.136794, "ttft_error_pct": 0.88, "ttft_ms": 143.73818973006, "ttft_relative": 1.008809, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5674.584884850035, "interactivity_tok_s": 164.0635766240351, "output_per_gpu": 78.96816399432743, "total_per_gpu": 709.8766987730797, "tpot_error_pct": 12.35, "tpot_ms": 6.095198096842546, "tpot_relative": 1.123547, "ttft_error_pct": 49.16, "ttft_ms": 72.43595822499972, "ttft_relative": 0.508383, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5128.29423712501, "interactivity_tok_s": 184.33311915844885, "output_per_gpu": 87.24852151386395, "total_per_gpu": 784.3121745814279, "tpot_ms": 5.424961095246379, "tpot_relative": 1.0, "ttft_ms": 142.48304342581832, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.38, "tpot_shape_error_pct": 4.82, "ttft_mape_pct": 26.7, "ttft_shape_error_pct": 38.64}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.07, "tpot_shape_error_pct": 4.46, "ttft_mape_pct": 32.99, "ttft_shape_error_pct": 25.55}, "framework": "vllm", "id": "fdab4dab9a7d721f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 9067.524779715146, "interactivity_tok_s": 115.83413139046456, "output_per_gpu": 451.2808180192863, "total_per_gpu": null, "tpot_error_pct": 13.0, "tpot_ms": 8.633034046149197, "tpot_relative": 1.130021, "ttft_error_pct": 10.85, "ttft_ms": 235.93095050451907, "ttft_relative": 1.108527, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7856.307258050011, "interactivity_tok_s": 119.27459546984784, "output_per_gpu": 458.16702308478705, "total_per_gpu": 4118.6482929688145, "tpot_error_pct": 9.74, "tpot_ms": 8.384015020640302, "tpot_relative": 1.097426, "ttft_error_pct": 29.22, "ttft_ms": 150.64834822499992, "ttft_relative": 0.707825, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 7234.491214068839, "interactivity_tok_s": 130.89499681039862, "output_per_gpu": 496.59770353964075, "total_per_gpu": 4464.1171907243, "tpot_ms": 7.6397114050776125, "tpot_relative": 1.0, "ttft_ms": 212.8329026629217, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12458.584719851007, "interactivity_tok_s": 83.86670203117578, "output_per_gpu": 656.8964440206394, "total_per_gpu": null, "tpot_error_pct": 19.41, "tpot_ms": 11.923683366352833, "tpot_relative": 1.56075, "ttft_error_pct": 4.19, "ttft_ms": 260.6566360720586, "ttft_relative": 1.224701, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10853.982575487511, "interactivity_tok_s": 86.73053846010615, "output_per_gpu": 672.7553425208969, "total_per_gpu": 5984.469812870208, "tpot_error_pct": 15.47, "tpot_ms": 11.529964159740283, "tpot_relative": 1.509215, "ttft_error_pct": 33.99, "ttft_ms": 179.5785042624994, "ttft_relative": 0.843753, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 9515.449095147778, "interactivity_tok_s": 100.14547740976678, "output_per_gpu": 765.2969576819469, "total_per_gpu": 6807.670265341315, "tpot_ms": 9.98547339195643, "tpot_relative": 1.307049, "ttft_ms": 272.05218268791214, "ttft_relative": 1.278243, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16888.045457159362, "interactivity_tok_s": 61.620274054994496, "output_per_gpu": 969.2062969348002, "total_per_gpu": null, "tpot_error_pct": 15.52, "tpot_ms": 16.228425065223274, "tpot_relative": 2.12422, "ttft_error_pct": 17.63, "ttft_ms": 286.3666154359524, "ttft_relative": 1.3455, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14647.170802487448, "interactivity_tok_s": 63.48656231814906, "output_per_gpu": 975.7124246264717, "total_per_gpu": 8805.560414284677, "tpot_error_pct": 12.13, "tpot_ms": 15.751364753201127, "tpot_relative": 2.061775, "ttft_error_pct": 31.3, "ttft_ms": 238.84652189374964, "ttft_relative": 1.122226, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 13199.158165124663, "interactivity_tok_s": 71.18468411303289, "output_per_gpu": 1082.4955473136563, "total_per_gpu": 9769.251369032889, "tpot_ms": 14.047965688969244, "tpot_relative": 1.838808, "ttft_ms": 347.6454723441748, "ttft_relative": 1.63342, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23790.344016867664, "interactivity_tok_s": 43.57823573502052, "output_per_gpu": 1376.0204550547796, "total_per_gpu": null, "tpot_error_pct": 5.6, "tpot_ms": 22.947234626030898, "tpot_relative": 3.003678, "ttft_error_pct": 74.12, "ttft_ms": 315.32299443805516, "ttft_relative": 1.481552, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21103.369085331087, "interactivity_tok_s": 44.66367977212452, "output_per_gpu": 1369.5969259832661, "total_per_gpu": 12247.37180137135, "tpot_error_pct": 3.03, "tpot_ms": 22.389556908477562, "tpot_relative": 2.930681, "ttft_error_pct": 67.97, "ttft_ms": 390.1995531374997, "ttft_relative": 1.833361, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 21332.25007585279, "interactivity_tok_s": 46.016452559026256, "output_per_gpu": 1350.6312533615926, "total_per_gpu": 12077.77471798569, "tpot_ms": 21.73135790329077, "tpot_relative": 2.844526, "ttft_ms": 1218.184306276089, "ttft_relative": 5.723665, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 164345.0884395029, "interactivity_tok_s": 42.73187570492786, "output_per_gpu": 1371.6289831845136, "total_per_gpu": 12341.111569653578, "tpot_error_pct": 5.01, "tpot_ms": 23.401734267533676, "tpot_relative": 3.06317, "ttft_error_pct": 2.47, "ttft_ms": 142779.94545713966, "ttft_relative": 670.854664, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 166939.21795659533, "interactivity_tok_s": 44.87142871232147, "output_per_gpu": 1351.0519602972724, "total_per_gpu": 12155.971609550683, "tpot_ms": 22.285896141421613, "tpot_relative": 2.917112, "ttft_ms": 146399.66525538397, "ttft_relative": 687.861996, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 33.33, "points": 1, "status_counts": {"failed": 2, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.63, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.74, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.5, "tpot_shape_error_pct": 0.52, "ttft_mape_pct": 11.79, "ttft_shape_error_pct": 15.53}, "framework": "vllm", "id": "d3bc8038be8473a9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 36716.87056105525, "interactivity_tok_s": 28.09737237998627, "output_per_gpu": 1783.1585045116744, "total_per_gpu": null, "tpot_error_pct": 6.63, "tpot_ms": 35.590516667398376, "tpot_relative": 1.066287, "ttft_error_pct": 62.74, "ttft_ms": 307.7720103067133, "ttft_relative": 0.372593, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32343.033995279042, "interactivity_tok_s": 29.046423900562328, "output_per_gpu": 1791.4068569255653, "total_per_gpu": 16172.967722317979, "tpot_error_pct": 3.14, "tpot_ms": 34.42764601327189, "tpot_relative": 1.031447, "ttft_error_pct": 20.07, "ttft_ms": 660.2683768328118, "ttft_relative": 0.799331, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 31543.64451484653, "interactivity_tok_s": 29.959859414187267, "output_per_gpu": 1837.5668235898465, "total_per_gpu": 16589.703679333306, "tpot_ms": 33.37799374073356, "tpot_relative": 1.0, "ttft_ms": 826.0266710492942, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 57742.4008799264, "interactivity_tok_s": 20.422197833836616, "output_per_gpu": 2004.4339095967046, "total_per_gpu": 18034.718437414496, "tpot_error_pct": 3.72, "tpot_ms": 48.96632615825243, "tpot_relative": 1.467024, "ttft_error_pct": 12.71, "ttft_ms": 12533.961982198785, "ttft_relative": 15.173798, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 57925.37078624537, "interactivity_tok_s": 21.18291576752596, "output_per_gpu": 1991.1554185088464, "total_per_gpu": 17915.246377549265, "tpot_ms": 47.207854243230756, "tpot_relative": 1.414341, "ttft_ms": 14358.435714773623, "ttft_relative": 17.382533, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 113978.78945285098, "interactivity_tok_s": 20.20761912024445, "output_per_gpu": 2003.8453023900347, "total_per_gpu": 18034.75329903212, "tpot_error_pct": 3.63, "tpot_ms": 49.486285051670315, "tpot_relative": 1.482602, "ttft_error_pct": 2.61, "ttft_ms": 68304.90064071739, "ttft_relative": 82.690914, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 114211.2237208717, "interactivity_tok_s": 20.941140021103774, "output_per_gpu": 1997.499489510987, "total_per_gpu": 17977.640522103207, "tpot_ms": 47.75289210578955, "tpot_relative": 1.43067, "ttft_ms": 70133.33628584283, "ttft_relative": 84.904445, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 88.1, "points": 37, "status_counts": {"failed": 5, "success": 37, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.56, "tpot_shape_error_pct": 4.06, "ttft_mape_pct": 44.19, "ttft_shape_error_pct": 39.58}, "aisimulate": {"coverage_pct": 100.0, "points": 42, "status_counts": {"failed": 0, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.72, "tpot_shape_error_pct": 4.51, "ttft_mape_pct": 33.23, "ttft_shape_error_pct": 19.14}, "gpu": "b300", "precisions": ["fp4"], "rows": 42, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.3, "tpot_shape_error_pct": 14.48, "ttft_mape_pct": 73.62, "ttft_shape_error_pct": 84.22}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.8, "tpot_shape_error_pct": 12.04, "ttft_mape_pct": 51.05, "ttft_shape_error_pct": 54.62}, "framework": "dynamo-vllm", "id": "cb47c94ee623863b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10643.336, "interactivity_tok_s": 98.34775767112511, "output_per_gpu": 579.5676000000001, "total_per_gpu": null, "tpot_error_pct": 13.11, "tpot_ms": 10.168, "tpot_relative": 1.13114, "ttft_error_pct": 54.43, "ttft_ms": 241.472, "ttft_relative": 0.455691, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8805.19968687188, "interactivity_tok_s": 109.4057984702874, "output_per_gpu": 649.5236218896947, "total_per_gpu": 5808.247039796312, "tpot_error_pct": 1.68, "tpot_ms": 9.140283366896515, "tpot_relative": 1.016812, "ttft_error_pct": 32.65, "ttft_ms": 356.8720071218748, "ttft_relative": 0.673467, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 8839.23948716656, "interactivity_tok_s": 111.24509509429811, "output_per_gpu": 810.9679886450816, "total_per_gpu": 5806.426850561352, "tpot_ms": 8.989160368395021, "tpot_relative": 1.0, "ttft_ms": 529.902934659367, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18451.895, "interactivity_tok_s": 56.17661929105107, "output_per_gpu": 1324.376, "total_per_gpu": null, "tpot_error_pct": 29.49, "tpot_ms": 17.801, "tpot_relative": 1.980274, "ttft_error_pct": 92.81, "ttft_ms": 241.472, "ttft_relative": 0.455691, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15418.856599803014, "interactivity_tok_s": 63.85093954199707, "output_per_gpu": 1476.8555731745323, "total_per_gpu": 13333.1729881112, "tpot_error_pct": 13.92, "tpot_ms": 15.66147667008508, "tpot_relative": 1.742262, "ttft_error_pct": 69.44, "ttft_ms": 1026.6049525390588, "ttft_relative": 1.937345, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 15993.034874697332, "interactivity_tok_s": 72.74175279909889, "output_per_gpu": 1779.3296764030335, "total_per_gpu": 12856.984720937377, "tpot_ms": 13.74726290637841, "tpot_relative": 1.529316, "ttft_ms": 3359.03015136455, "ttft_relative": 6.338954, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.44, "tpot_shape_error_pct": 0.56, "ttft_mape_pct": 73.47, "ttft_shape_error_pct": 60.29}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.7, "tpot_shape_error_pct": 3.73, "ttft_mape_pct": 40.04, "ttft_shape_error_pct": 12.82}, "framework": "dynamo-vllm", "id": "95e828ee26ef6835", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 14081.639, "interactivity_tok_s": 73.91529307413704, "output_per_gpu": 726.0296666666667, "total_per_gpu": null, "tpot_error_pct": 22.39, "tpot_ms": 13.529, "tpot_relative": 1.223913, "ttft_error_pct": 51.56, "ttft_ms": 241.472, "ttft_relative": 0.484377, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11489.486059179699, "interactivity_tok_s": 82.41594896532105, "output_per_gpu": 832.540455447723, "total_per_gpu": 7504.334821376305, "tpot_error_pct": 9.77, "tpot_ms": 12.133573811311445, "tpot_relative": 1.097674, "ttft_error_pct": 35.95, "ttft_ms": 319.2867864718757, "ttft_relative": 0.640468, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 10673.764683925765, "interactivity_tok_s": 90.46587439329258, "output_per_gpu": 1343.3765206357325, "total_per_gpu": 8079.103286297869, "tpot_ms": 11.053891942198959, "tpot_relative": 1.0, "ttft_ms": 498.52078498370247, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18451.895, "interactivity_tok_s": 56.17661929105107, "output_per_gpu": 1103.6466666666668, "total_per_gpu": null, "tpot_error_pct": 23.17, "tpot_ms": 17.801, "tpot_relative": 1.610383, "ttft_error_pct": 66.75, "ttft_ms": 241.472, "ttft_relative": 0.484377, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15018.246813321777, "interactivity_tok_s": 63.27597061636602, "output_per_gpu": 1267.7922298317687, "total_per_gpu": 11445.732013588407, "tpot_error_pct": 9.35, "tpot_ms": 15.803787603083483, "tpot_relative": 1.429703, "ttft_error_pct": 31.81, "ttft_ms": 495.1971006507809, "ttft_relative": 0.993333, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 14008.49012310664, "interactivity_tok_s": 69.19270368257679, "output_per_gpu": 2039.4876410714119, "total_per_gpu": 12280.683463525622, "tpot_ms": 14.45239088484711, "tpot_relative": 1.307448, "ttft_ms": 726.171656207589, "ttft_relative": 1.456653, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25922.864, "interactivity_tok_s": 39.834289356277885, "output_per_gpu": 1565.1088333333335, "total_per_gpu": null, "tpot_error_pct": 21.46, "tpot_ms": 25.104, "tpot_relative": 2.271055, "ttft_error_pct": 82.12, "ttft_ms": 241.472, "ttft_relative": 0.484377, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20764.50044825111, "interactivity_tok_s": 46.17996700854142, "output_per_gpu": 1838.6055444936062, "total_per_gpu": 16542.692255232778, "tpot_error_pct": 4.77, "tpot_ms": 21.654411312486225, "tpot_relative": 1.958985, "ttft_error_pct": 39.23, "ttft_ms": 820.9474678253888, "ttft_relative": 1.646767, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 20386.33235169218, "interactivity_tok_s": 48.381448299402855, "output_per_gpu": 2801.0671157171514, "total_per_gpu": 16808.527768929645, "tpot_ms": 20.669079474670095, "tpot_relative": 1.869846, "ttft_ms": 1350.8220087597692, "ttft_relative": 2.70966, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 40085.276, "interactivity_tok_s": 25.67525932011913, "output_per_gpu": 2017.6213333333333, "total_per_gpu": null, "tpot_error_pct": 22.75, "tpot_ms": 38.948, "tpot_relative": 3.523465, "ttft_error_pct": 93.43, "ttft_ms": 241.472, "ttft_relative": 0.484377, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31795.77590211727, "interactivity_tok_s": 30.625280467180403, "output_per_gpu": 2406.7424914844264, "total_per_gpu": 21660.857270992597, "tpot_error_pct": 2.91, "tpot_ms": 32.65276218683615, "tpot_relative": 2.953961, "ttft_error_pct": 53.16, "ttft_ms": 1721.8461320580125, "ttft_relative": 3.45391, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 32899.58264573925, "interactivity_tok_s": 31.515148522542944, "output_per_gpu": 3495.0123619853703, "total_per_gpu": 20982.217392639624, "tpot_ms": 31.730772243852666, "tpot_relative": 2.870552, "ttft_ms": 3675.9153839705505, "ttft_relative": 7.373645, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.78, "tpot_shape_error_pct": 2.15, "ttft_mape_pct": 76.99, "ttft_shape_error_pct": 19.82}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.0, "tpot_shape_error_pct": 1.3, "ttft_mape_pct": 32.39, "ttft_shape_error_pct": 1.5}, "framework": "dynamo-vllm", "id": "c375590855609983", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 31121.75, "interactivity_tok_s": 33.12794010468429, "output_per_gpu": 1464.345625, "total_per_gpu": null, "tpot_error_pct": 11.58, "tpot_ms": 30.186, "tpot_relative": 1.115758, "ttft_error_pct": 74.45, "ttft_ms": 241.472, "ttft_relative": 0.255463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24893.842344996192, "interactivity_tok_s": 37.962608558910354, "output_per_gpu": 1729.695703401355, "total_per_gpu": 15557.429411264411, "tpot_error_pct": 2.63, "tpot_ms": 26.34170932822492, "tpot_relative": 0.973663, "ttft_error_pct": 32.89, "ttft_ms": 634.3495910914066, "ttft_relative": 0.671105, "unavailable_metrics": {}}, "concurrency": 384, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 25862.243105988804, "interactivity_tok_s": 36.96277498252833, "output_per_gpu": 3326.7633009002093, "total_per_gpu": 14967.631998324878, "tpot_ms": 27.054245804669232, "tpot_relative": 1.0, "ttft_ms": 945.2314728341046, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37385.579, "interactivity_tok_s": 27.541380924839572, "output_per_gpu": 1623.203875, "total_per_gpu": null, "tpot_error_pct": 13.97, "tpot_ms": 36.309, "tpot_relative": 1.342081, "ttft_error_pct": 79.52, "ttft_ms": 241.472, "ttft_relative": 0.255463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29742.942561077634, "interactivity_tok_s": 31.825399367441822, "output_per_gpu": 1928.9320679707264, "total_per_gpu": 17360.52874688063, "tpot_error_pct": 1.37, "tpot_ms": 31.421443874260536, "tpot_relative": 1.161424, "ttft_error_pct": 31.88, "ttft_ms": 803.0868425103517, "ttft_relative": 0.849619, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 30517.112263287698, "interactivity_tok_s": 31.390103137103882, "output_per_gpu": 3760.37909172925, "total_per_gpu": 16931.504829480655, "tpot_ms": 31.857174716255557, "tpot_relative": 1.17753, "ttft_ms": 1178.9434890128896, "ttft_relative": 1.247254, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.62, "tpot_shape_error_pct": null, "ttft_mape_pct": 74.1, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.42, "tpot_shape_error_pct": null, "ttft_mape_pct": 32.29, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5f704b767d15ea98", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 22214.489, "interactivity_tok_s": 46.557102285953725, "output_per_gpu": 1371.9825, "total_per_gpu": null, "tpot_error_pct": 10.62, "tpot_ms": 21.479, "tpot_relative": 1.1062, "ttft_error_pct": 74.1, "ttft_ms": 241.472, "ttft_relative": 0.25905, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18259.590728640324, "interactivity_tok_s": 52.243450467965914, "output_per_gpu": 1565.2430089902534, "total_per_gpu": 14078.289941956675, "tpot_error_pct": 1.42, "tpot_ms": 19.14115532267857, "tpot_relative": 0.985797, "ttft_error_pct": 32.29, "ttft_ms": 631.1346142960923, "ttft_relative": 0.677077, "unavailable_metrics": {}}, "concurrency": 384, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 18823.868768808567, "interactivity_tok_s": 51.501451989020296, "output_per_gpu": 2258.3034888989605, "total_per_gpu": 13543.051198894895, "tpot_ms": 19.41692828802559, "tpot_relative": 1.0, "ttft_ms": 932.1459666630063, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.46, "tpot_shape_error_pct": null, "ttft_mape_pct": 3.53, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.67, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.11, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "5fc8d678e734f64a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6598.394, "interactivity_tok_s": 160.926939169617, "output_per_gpu": 65.84088888888888, "total_per_gpu": null, "tpot_error_pct": 11.46, "tpot_ms": 6.214, "tpot_relative": 1.114559, "ttft_error_pct": 3.53, "ttft_ms": 241.472, "ttft_relative": 0.964652, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5403.770373999991, "interactivity_tok_s": 174.69331136554973, "output_per_gpu": 73.62908456800913, "total_per_gpu": 661.881558884626, "tpot_error_pct": 2.67, "tpot_ms": 5.7243176180196, "tpot_relative": 1.026728, "ttft_error_pct": 43.11, "ttft_ms": 142.4110558250002, "ttft_relative": 0.568915, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5374.769606275095, "interactivity_tok_s": 179.36256948076965, "output_per_gpu": 82.99924239842663, "total_per_gpu": 663.8536213690517, "tpot_ms": 5.575299254994309, "tpot_relative": 1.0, "ttft_ms": 250.3204449502846, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.16, "tpot_shape_error_pct": 5.73, "ttft_mape_pct": 17.79, "ttft_shape_error_pct": 27.7}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.77, "tpot_shape_error_pct": 4.79, "ttft_mape_pct": 40.85, "ttft_shape_error_pct": 3.02}, "framework": "dynamo-vllm", "id": "64ae572986963121", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6550.313, "interactivity_tok_s": 162.15339711366954, "output_per_gpu": 59.7278, "total_per_gpu": null, "tpot_error_pct": 8.5, "tpot_ms": 6.167, "tpot_relative": 1.084962, "ttft_error_pct": 10.22, "ttft_ms": 241.472, "ttft_relative": 1.102241, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5446.40571535001, "interactivity_tok_s": 175.61965532773962, "output_per_gpu": 68.4442834732961, "total_per_gpu": 609.9390092424621, "tpot_error_pct": 0.18, "tpot_ms": 5.694123463195564, "tpot_relative": 1.001769, "ttft_error_pct": 42.16, "ttft_ms": 126.71330830000011, "ttft_relative": 0.578405, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5529.351879563183, "interactivity_tok_s": 175.93031577169933, "output_per_gpu": 84.30125289306145, "total_per_gpu": 601.5751828475555, "tpot_ms": 5.684068692843573, "tpot_relative": 1.0, "ttft_ms": 219.07368609681726, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7062.836, "interactivity_tok_s": 149.97000599880025, "output_per_gpu": 110.4118, "total_per_gpu": null, "tpot_error_pct": 11.73, "tpot_ms": 6.668, "tpot_relative": 1.173103, "ttft_error_pct": 1.7, "ttft_ms": 241.472, "ttft_relative": 1.102241, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5774.00257087498, "interactivity_tok_s": 163.2127911317047, "output_per_gpu": 124.11431548356579, "total_per_gpu": 1115.7136761123531, "tpot_error_pct": 2.66, "tpot_ms": 6.126970766605229, "tpot_relative": 1.07792, "ttft_error_pct": 41.97, "ttft_ms": 142.55516089999716, "ttft_relative": 0.650718, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 5730.820269818651, "interactivity_tok_s": 167.55542460298608, "output_per_gpu": 156.44377632809997, "total_per_gpu": 1126.1571552877817, "tpot_ms": 5.9681744256830145, "tpot_relative": 1.049983, "ttft_ms": 245.64437931985594, "ttft_relative": 1.121287, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7974.329, "interactivity_tok_s": 132.29263130043657, "output_per_gpu": 195.0106, "total_per_gpu": null, "tpot_error_pct": 16.43, "tpot_ms": 7.559, "tpot_relative": 1.329857, "ttft_error_pct": 22.94, "ttft_ms": 241.472, "ttft_relative": 1.102241, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6593.058188549976, "interactivity_tok_s": 144.27066312218446, "output_per_gpu": 220.4155882819611, "total_per_gpu": 1960.6985645282366, "tpot_error_pct": 6.76, "tpot_ms": 6.9314161199431705, "tpot_relative": 1.219446, "ttft_error_pct": 41.97, "ttft_ms": 181.82781659999966, "ttft_relative": 0.829985, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 6318.02729566698, "interactivity_tok_s": 154.02767680727482, "output_per_gpu": 287.214879318421, "total_per_gpu": 2045.9126790244384, "tpot_ms": 6.492339693282768, "tpot_relative": 1.142199, "ttft_ms": 313.3577437110944, "ttft_relative": 1.430376, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8845.925, "interactivity_tok_s": 118.89192723814054, "output_per_gpu": 350.2654, "total_per_gpu": null, "tpot_error_pct": 15.97, "tpot_ms": 8.411, "tpot_relative": 1.47975, "ttft_error_pct": 36.29, "ttft_ms": 241.472, "ttft_relative": 1.102241, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7232.056663587535, "interactivity_tok_s": 130.70153338053115, "output_per_gpu": 390.4200525361723, "total_per_gpu": 3523.4432531404614, "tpot_error_pct": 5.49, "tpot_ms": 7.651019648626071, "tpot_relative": 1.346046, "ttft_error_pct": 37.29, "ttft_ms": 237.67236113124954, "ttft_relative": 1.084897, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26857293503", "measured": {"e2e_ms": 7005.0544524608995, "interactivity_tok_s": 137.88072277473998, "output_per_gpu": 506.23200216194124, "total_per_gpu": 3657.297107142942, "tpot_ms": 7.2526454741155595, "tpot_relative": 1.27596, "ttft_ms": 378.997390170116, "ttft_relative": 1.73, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.1, "tpot_shape_error_pct": null, "ttft_mape_pct": 44.95, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 73.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.63, "ttft_shape_error_pct": null}, "framework": "trt", "id": "f151dc825b4c4ee8", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 25115.036097620727, "interactivity_tok_s": 41.928173828581656, "output_per_gpu": 1310.8073130184807, "total_per_gpu": null, "tpot_error_pct": 11.1, "tpot_ms": 23.850311346455985, "tpot_relative": 1.111035, "ttft_error_pct": 44.95, "ttft_ms": 716.1675901962552, "ttft_relative": 0.55055, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34705.33746797878, "interactivity_tok_s": 26.91405416899324, "output_per_gpu": 835.7330029581707, "total_per_gpu": 7545.065951413842, "tpot_error_pct": 73.08, "tpot_ms": 37.155309033748836, "tpot_relative": 1.730831, "ttft_error_pct": 62.63, "ttft_ms": 486.16508613124944, "ttft_relative": 0.373736, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 21039.34630916583, "interactivity_tok_s": 46.58368753872872, "output_per_gpu": 1365.7305647428059, "total_per_gpu": 12329.92731694466, "tpot_ms": 21.466741961306102, "tpot_relative": 1.0, "ttft_ms": 1300.8231926745793, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.79, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.77, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.78, "tpot_shape_error_pct": null, "ttft_mape_pct": 19.48, "ttft_shape_error_pct": null}, "framework": "trt", "id": "75aa62a7d955a8f2", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 4, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7708.613013039025, "interactivity_tok_s": 135.68071401790698, "output_per_gpu": 65.81214622379572, "total_per_gpu": null, "tpot_error_pct": 24.79, "tpot_ms": 7.370244232854061, "tpot_relative": 1.247914, "ttft_error_pct": 49.77, "ttft_ms": 168.85316282932027, "ttft_relative": 1.497665, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6972.588545875015, "interactivity_tok_s": 133.55637112899024, "output_per_gpu": 64.28756393660697, "total_per_gpu": 577.906859563812, "tpot_error_pct": 26.78, "tpot_ms": 7.487475075480965, "tpot_relative": 1.267763, "ttft_error_pct": 19.48, "ttft_ms": 90.78503389999965, "ttft_relative": 0.80523, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 5541.586073997314, "interactivity_tok_s": 169.31782673195917, "output_per_gpu": 80.94297728014055, "total_per_gpu": 727.6290924608314, "tpot_ms": 5.9060526543555465, "tpot_relative": 1.0, "ttft_ms": 112.74425442097709, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.38, "tpot_shape_error_pct": null, "ttft_mape_pct": 30.92, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.17, "tpot_shape_error_pct": null, "ttft_mape_pct": 29.96, "ttft_shape_error_pct": null}, "framework": "trt", "id": "36218a7a331e371a", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 2, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8197.838468469417, "interactivity_tok_s": 127.6323528676743, "output_per_gpu": 123.74798058628612, "total_per_gpu": null, "tpot_error_pct": 18.38, "tpot_ms": 7.8350040372347625, "tpot_relative": 1.183802, "ttft_error_pct": 30.92, "ttft_ms": 182.62933837825645, "ttft_relative": 1.309167, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7408.02616982504, "interactivity_tok_s": 125.72668720849727, "output_per_gpu": 121.15094094103902, "total_per_gpu": 1089.074706291196, "tpot_error_pct": 20.17, "tpot_ms": 7.953760829963352, "tpot_relative": 1.201745, "ttft_error_pct": 29.96, "ttft_ms": 97.70096082500059, "ttft_relative": 0.700363, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 6222.934890999932, "interactivity_tok_s": 151.091404964784, "output_per_gpu": 144.36366539397926, "total_per_gpu": 1297.7432553708738, "tpot_ms": 6.618510167623879, "tpot_relative": 1.0, "ttft_ms": 139.5004576748761, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.32, "tpot_shape_error_pct": 2.26, "ttft_mape_pct": 65.47, "ttft_shape_error_pct": 14.59}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.07, "tpot_shape_error_pct": 2.77, "ttft_mape_pct": 6.4, "ttft_shape_error_pct": 5.18}, "framework": "trt", "id": "ccd90e641e818819", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 11572.122669021624, "interactivity_tok_s": 91.13449159723932, "output_per_gpu": 349.4162481162207, "total_per_gpu": null, "tpot_error_pct": 35.35, "tpot_ms": 10.97279397156688, "tpot_relative": 1.353461, "ttft_error_pct": 83.3, "ttft_ms": 346.9544361087072, "ttft_relative": 1.832967, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10436.782932199998, "interactivity_tok_s": 89.63855311224934, "output_per_gpu": 344.71821206882106, "total_per_gpu": 3098.811141258791, "tpot_error_pct": 37.6, "tpot_ms": 11.15591411596923, "tpot_relative": 1.376048, "ttft_error_pct": 3.05, "ttft_ms": 183.50692700000096, "ttft_relative": 0.969471, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 7640.991942925029, "interactivity_tok_s": 123.34696618168614, "output_per_gpu": 470.9300655235157, "total_per_gpu": 4233.3804327885555, "tpot_ms": 8.10721196439507, "tpot_relative": 1.0, "ttft_ms": 189.28565162059385, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15402.304414251303, "interactivity_tok_s": 68.1110435319446, "output_per_gpu": 527.9825657105534, "total_per_gpu": null, "tpot_error_pct": 36.86, "tpot_ms": 14.68190690003145, "tpot_relative": 1.810969, "ttft_error_pct": 63.67, "ttft_ms": 382.71365551912953, "ttft_relative": 2.021884, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13835.45194587502, "interactivity_tok_s": 67.9690968652435, "output_per_gpu": 527.4300645380093, "total_per_gpu": 4691.734275643988, "tpot_error_pct": 37.14, "tpot_ms": 14.712568595439986, "tpot_relative": 1.814751, "ttft_error_pct": 7.86, "ttft_ms": 215.4483525125007, "ttft_relative": 1.138218, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 10164.835356795811, "interactivity_tok_s": 93.21454882989337, "output_per_gpu": 717.7772061918064, "total_per_gpu": 6384.960105594041, "tpot_ms": 10.727939066946444, "tpot_relative": 1.323259, "ttft_ms": 233.8298771515838, "ttft_relative": 1.235328, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20503.28065431573, "interactivity_tok_s": 51.02541654728347, "output_per_gpu": 797.5012394562493, "total_per_gpu": null, "tpot_error_pct": 30.75, "tpot_ms": 19.59807616804725, "tpot_relative": 2.417363, "ttft_error_pct": 49.44, "ttft_ms": 454.4487344033956, "ttft_relative": 2.400862, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18164.314677025053, "interactivity_tok_s": 51.14343406550659, "output_per_gpu": 785.5330443076735, "total_per_gpu": 7089.239108250787, "tpot_error_pct": 30.45, "tpot_ms": 19.5528520575908, "tpot_relative": 2.411785, "ttft_error_pct": 8.28, "ttft_ms": 278.91463404375037, "ttft_relative": 1.473512, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 14015.708882271429, "interactivity_tok_s": 66.7165133794523, "output_per_gpu": 1019.3223204637736, "total_per_gpu": 9199.128808786809, "tpot_ms": 14.988792869202683, "tpot_relative": 1.848822, "ttft_ms": 304.1036628826987, "ttft_relative": 1.606586, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 66.67, "points": 2, "status_counts": {"failed": 1, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.6, "tpot_shape_error_pct": 2.03, "ttft_mape_pct": 63.44, "ttft_shape_error_pct": 40.64}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.23, "tpot_shape_error_pct": 2.24, "ttft_mape_pct": 45.57, "ttft_shape_error_pct": 98.19}, "framework": "trt", "id": "b86b10419fffb87e", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 39589.64137008425, "interactivity_tok_s": 26.353035829701692, "output_per_gpu": 1665.0601207982688, "total_per_gpu": null, "tpot_error_pct": 17.79, "tpot_ms": 37.946292277754615, "tpot_relative": 1.177942, "ttft_error_pct": 54.12, "ttft_ms": 770.5843699412783, "ttft_relative": 0.458793, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42178.817149295544, "interactivity_tok_s": 22.160487124852125, "output_per_gpu": 1374.3716850400303, "total_per_gpu": 12407.94005822194, "tpot_error_pct": 40.08, "tpot_ms": 45.125361837309924, "tpot_relative": 1.400797, "ttft_error_pct": 61.91, "ttft_ms": 639.7906697578125, "ttft_relative": 0.38092, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 31294.09185685072, "interactivity_tok_s": 31.042353450404345, "output_per_gpu": 1844.4091548943782, "total_per_gpu": 16651.476806362367, "tpot_ms": 32.21405237839576, "tpot_relative": 1.0, "ttft_ms": 1679.5922134642751, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 62265.36073069455, "interactivity_tok_s": 16.647460330848908, "output_per_gpu": 2113.99172875757, "total_per_gpu": null, "tpot_error_pct": 15.41, "tpot_ms": 60.06922257967061, "tpot_relative": 1.86469, "ttft_error_pct": 72.77, "ttft_ms": 814.54603169152, "ttft_relative": 0.484967, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 66293.7318697289, "interactivity_tok_s": 14.161510663335243, "output_per_gpu": 1758.2886940836927, "total_per_gpu": 15820.048432461805, "tpot_error_pct": 35.67, "tpot_ms": 70.61393546022197, "tpot_relative": 2.192023, "ttft_error_pct": 62.38, "ttft_ms": 1125.274296150776, "ttft_relative": 0.669969, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 50930.73766863622, "interactivity_tok_s": 19.21246452398094, "output_per_gpu": 2273.3000116544185, "total_per_gpu": 20453.817627844594, "tpot_ms": 52.04954308448055, "tpot_relative": 1.61574, "ttft_ms": 2991.1311309953817, "ttft_relative": 1.780867, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 124831.30554835915, "interactivity_tok_s": 10.642040682337448, "output_per_gpu": 1853.5173724460578, "total_per_gpu": 16681.791008350207, "tpot_error_pct": 41.94, "tpot_ms": 93.96694016211532, "tpot_relative": 2.916955, "ttft_error_pct": 12.43, "ttft_ms": 38134.20778099051, "ttft_relative": 22.704444, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27439020618", "measured": {"e2e_ms": 94890.0606658234, "interactivity_tok_s": 15.105770519613467, "output_per_gpu": 2431.3112717543804, "total_per_gpu": 21881.978078321517, "tpot_ms": 66.19986704429219, "tpot_relative": 2.055, "ttft_ms": 33919.30846407151, "ttft_relative": 20.194966, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.08, "tpot_shape_error_pct": null, "ttft_mape_pct": 26.47, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.61, "tpot_shape_error_pct": null, "ttft_mape_pct": 37.92, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "9512165938b35768", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6248.397514893741, "interactivity_tok_s": 167.45052707311666, "output_per_gpu": 81.86098896249537, "total_per_gpu": null, "tpot_error_pct": 11.08, "tpot_ms": 5.971913122515008, "tpot_relative": 1.110827, "ttft_error_pct": 26.47, "ttft_ms": 139.13039056088735, "ttft_relative": 1.264687, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5484.594758899993, "interactivity_tok_s": 169.69348226728218, "output_per_gpu": 81.71608723648468, "total_per_gpu": 734.5788898961321, "tpot_error_pct": 9.61, "tpot_ms": 5.892978249010837, "tpot_relative": 1.096144, "ttft_error_pct": 37.92, "ttft_ms": 68.29318032499987, "ttft_relative": 0.620781, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5051.284683075937, "interactivity_tok_s": 186.0085058460317, "output_per_gpu": 88.42193616848446, "total_per_gpu": 794.8604725179712, "tpot_ms": 5.376098235140648, "tpot_relative": 1.0, "ttft_ms": 110.0117302245053, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 87.5, "points": 7, "status_counts": {"failed": 1, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.29, "tpot_shape_error_pct": 5.07, "ttft_mape_pct": 37.58, "ttft_shape_error_pct": 46.09}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.08, "tpot_shape_error_pct": 5.17, "ttft_mape_pct": 31.65, "ttft_shape_error_pct": 15.43}, "framework": "vllm", "id": "730df049d54d9c55", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 8181.138471688436, "interactivity_tok_s": 127.77545440521305, "output_per_gpu": 250.08744285167236, "total_per_gpu": null, "tpot_error_pct": 0.51, "tpot_ms": 7.826229260188813, "tpot_relative": 1.005071, "ttft_error_pct": 6.02, "ttft_ms": 174.90593851527964, "ttft_relative": 1.060154, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7170.611399050002, "interactivity_tok_s": 129.99118044761613, "output_per_gpu": 250.66750860081706, "total_per_gpu": 2253.351407638183, "tpot_error_pct": 1.21, "tpot_ms": 7.692829594719929, "tpot_relative": 0.98794, "ttft_error_pct": 39.28, "ttft_ms": 100.18424190000034, "ttft_relative": 0.607245, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 7322.03515345027, "interactivity_tok_s": 128.4234257084344, "output_per_gpu": 244.51833088979947, "total_per_gpu": 2198.07396730185, "tpot_ms": 7.786741355664706, "tpot_relative": 1.0, "ttft_ms": 164.98166890196444, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9603.776773992255, "interactivity_tok_s": 108.70036704976512, "output_per_gpu": 426.08237324730845, "total_per_gpu": null, "tpot_error_pct": 3.0, "tpot_ms": 9.199600950217405, "tpot_relative": 1.181444, "ttft_error_pct": 5.44, "ttft_ms": 192.58500191984967, "ttft_relative": 1.167312, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8462.155364774997, "interactivity_tok_s": 110.93477327206912, "output_per_gpu": 430.3575621789918, "total_per_gpu": 3828.229486740334, "tpot_error_pct": 0.92, "tpot_ms": 9.01430606927447, "tpot_relative": 1.157648, "ttft_error_pct": 41.58, "ttft_ms": 118.98080052499998, "ttft_relative": 0.721176, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 8471.590816437038, "interactivity_tok_s": 111.95627336579496, "output_per_gpu": 429.50767623554066, "total_per_gpu": 3820.6693583377732, "tpot_ms": 8.932058650547415, "tpot_relative": 1.147086, "ttft_ms": 203.66537741192587, "ttft_relative": 1.234473, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12445.643881122029, "interactivity_tok_s": 83.61847108214268, "output_per_gpu": 657.5794774598819, "total_per_gpu": null, "tpot_error_pct": 4.36, "tpot_ms": 11.959080177603932, "tpot_relative": 1.535826, "ttft_error_pct": 17.4, "ttft_ms": 211.50485943320632, "ttft_relative": 1.28199, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10823.17822101879, "interactivity_tok_s": 85.75178021523132, "output_per_gpu": 656.9480225037928, "total_per_gpu": 5928.791470926045, "tpot_error_pct": 1.76, "tpot_ms": 11.661565479924333, "tpot_relative": 1.497618, "ttft_error_pct": 38.83, "ttft_ms": 156.6469540875001, "ttft_relative": 0.949481, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 10738.818854962756, "interactivity_tok_s": 87.26168775520098, "output_per_gpu": 660.6697666265012, "total_per_gpu": 5962.379280091807, "tpot_ms": 11.45978293252067, "tpot_relative": 1.471705, "ttft_ms": 256.0718106943568, "ttft_relative": 1.552123, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15924.46808897837, "interactivity_tok_s": 65.18751301759515, "output_per_gpu": 1027.8522276878189, "total_per_gpu": null, "tpot_error_pct": 6.62, "tpot_ms": 15.340361270264813, "tpot_relative": 1.970062, "ttft_error_pct": 36.12, "ttft_ms": 231.27850949746684, "ttft_relative": 1.401844, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14106.836349056297, "interactivity_tok_s": 66.6209470666288, "output_per_gpu": 1023.1687425146062, "total_per_gpu": 9149.500679640892, "tpot_error_pct": 8.63, "tpot_ms": 15.01029396955108, "tpot_relative": 1.927673, "ttft_error_pct": 39.25, "ttft_ms": 219.95523470000066, "ttft_relative": 1.33321, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 15561.055152590961, "interactivity_tok_s": 60.874549601004006, "output_per_gpu": 927.0227722691648, "total_per_gpu": 8289.73280015757, "tpot_ms": 16.427226263757145, "tpot_relative": 2.109641, "ttft_ms": 362.0608525413445, "ttft_relative": 2.194552, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23757.178449656174, "interactivity_tok_s": 43.52735509233886, "output_per_gpu": 1377.941411240011, "total_per_gpu": null, "tpot_error_pct": 2.84, "tpot_ms": 22.97405844849984, "tpot_relative": 2.950407, "ttft_error_pct": 52.63, "ttft_ms": 254.71665684083928, "ttft_relative": 1.543909, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21133.256073974924, "interactivity_tok_s": 44.34882230043528, "output_per_gpu": 1370.2542701733244, "total_per_gpu": 12351.167761897348, "tpot_error_pct": 4.64, "tpot_ms": 22.54851308622428, "tpot_relative": 2.895757, "ttft_error_pct": 34.35, "ttft_ms": 353.03465107343794, "ttft_relative": 2.139842, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 22333.14127644635, "interactivity_tok_s": 42.29048383359043, "output_per_gpu": 1295.850701696264, "total_per_gpu": 11680.510515029036, "tpot_ms": 23.645981538894606, "tpot_relative": 3.036698, "ttft_ms": 537.714548846634, "ttft_relative": 3.259238, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34997.61332474388, "interactivity_tok_s": 29.468460620250056, "output_per_gpu": 1870.7561396396777, "total_per_gpu": null, "tpot_error_pct": 5.77, "tpot_ms": 33.93458561974638, "tpot_relative": 4.357996, "ttft_error_pct": 66.36, "ttft_ms": 282.53223574333543, "ttft_relative": 1.712507, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31327.88375868761, "interactivity_tok_s": 29.965784402663967, "output_per_gpu": 1848.7074332017446, "total_per_gpu": 16690.28201471458, "tpot_error_pct": 7.33, "tpot_ms": 33.37139407273783, "tpot_relative": 4.285669, "ttft_error_pct": 26.21, "ttft_ms": 619.8087189632815, "ttft_relative": 3.756834, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 33979.59016447188, "interactivity_tok_s": 27.768781786006855, "output_per_gpu": 1705.170300384525, "total_per_gpu": 15394.417031820067, "tpot_ms": 36.011662582328924, "tpot_relative": 4.624741, "ttft_ms": 839.9487186664146, "ttft_relative": 5.091164, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 57537.96690586734, "interactivity_tok_s": 17.880360342139966, "output_per_gpu": 2275.784269788775, "total_per_gpu": null, "tpot_error_pct": 6.96, "tpot_ms": 55.92728451021348, "tpot_relative": 7.182373, "ttft_error_pct": 79.1, "ttft_ms": 324.3548519189494, "ttft_relative": 1.966005, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51562.69344601596, "interactivity_tok_s": 18.30667010003637, "output_per_gpu": 2258.788005326369, "total_per_gpu": 20323.2471227083, "tpot_error_pct": 9.13, "tpot_ms": 54.624898713721464, "tpot_relative": 7.015117, "ttft_error_pct": 24.22, "ttft_ms": 1176.0955136187506, "ttft_relative": 7.128644, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 57003.53892093426, "interactivity_tok_s": 16.635190645544018, "output_per_gpu": 2044.2115302152754, "total_per_gpu": 18392.614092906853, "tpot_ms": 60.11352808077765, "tpot_relative": 7.719985, "ttft_ms": 1552.0683540269515, "ttft_relative": 9.40752, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 99086.10921354136, "interactivity_tok_s": 12.838706639458033, "output_per_gpu": 2338.2662987602516, "total_per_gpu": 21044.56656174248, "tpot_error_pct": 6.99, "tpot_ms": 77.8894656667857, "tpot_relative": 10.002832, "ttft_error_pct": 9.53, "ttft_ms": 27204.67399438292, "ttft_relative": 164.895132, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "24565681098", "measured": {"e2e_ms": 107368.05603379999, "interactivity_tok_s": 11.941139395905036, "output_per_gpu": 2159.0612303739103, "total_per_gpu": 19431.70746988193, "tpot_ms": 83.74410237124684, "tpot_relative": 10.754705, "ttft_ms": 30069.537978220294, "ttft_relative": 182.259873, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.22, "tpot_shape_error_pct": 1.19, "ttft_mape_pct": 16.19, "ttft_shape_error_pct": 10.49}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.6, "tpot_shape_error_pct": 1.31, "ttft_mape_pct": 37.45, "ttft_shape_error_pct": 4.32}, "framework": "vllm", "id": "bc967f93222811a3", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6725.836616547127, "interactivity_tok_s": 155.58308497580012, "output_per_gpu": 152.100037262752, "total_per_gpu": null, "tpot_error_pct": 9.57, "tpot_ms": 6.427433934450799, "tpot_relative": 1.095667, "ttft_error_pct": 22.62, "ttft_ms": 150.57170160395955, "ttft_relative": 1.226156, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5895.973018849989, "interactivity_tok_s": 157.99090252971408, "output_per_gpu": 152.1456522526435, "total_per_gpu": 1367.6986761594449, "tpot_error_pct": 7.9, "tpot_ms": 6.329478368616353, "tpot_relative": 1.078969, "ttft_error_pct": 36.07, "ttft_ms": 78.50396617499962, "ttft_relative": 0.639284, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 5514.4867812989105, "interactivity_tok_s": 170.4672583550082, "output_per_gpu": 162.6083313251489, "total_per_gpu": 1461.7519212221746, "tpot_ms": 5.866229149514686, "tpot_relative": 1.0, "ttft_ms": 122.7998054746422, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7837.650528449706, "interactivity_tok_s": 133.34476505027445, "output_per_gpu": 261.0476178509455, "total_per_gpu": null, "tpot_error_pct": 10.87, "tpot_ms": 7.499357021049712, "tpot_relative": 1.278395, "ttft_error_pct": 9.76, "ttft_ms": 165.8082959158512, "ttft_relative": 1.350233, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6935.141110762495, "interactivity_tok_s": 135.25420958227866, "output_per_gpu": 262.37573467513715, "total_per_gpu": 2333.953466514804, "tpot_error_pct": 9.31, "tpot_ms": 7.3934852237754125, "tpot_relative": 1.260347, "ttft_error_pct": 38.83, "ttft_ms": 92.40605610000037, "ttft_relative": 0.752494, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 6411.227284162396, "interactivity_tok_s": 147.84087868463178, "output_per_gpu": 283.67735098156004, "total_per_gpu": 2523.4411921320434, "tpot_ms": 6.764029062172715, "tpot_relative": 1.153045, "ttft_ms": 151.06642724977064, "ttft_relative": 1.230185, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 71.43, "points": 5, "status_counts": {"failed": 2, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.8, "tpot_shape_error_pct": 3.91, "ttft_mape_pct": 24.1, "ttft_shape_error_pct": 36.49}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.78, "tpot_shape_error_pct": 5.12, "ttft_mape_pct": 22.96, "ttft_shape_error_pct": 12.48}, "framework": "vllm", "id": "1a4e4fb7745623fe", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8890.044939678004, "interactivity_tok_s": 117.96148987938548, "output_per_gpu": 460.29013663773577, "total_per_gpu": null, "tpot_error_pct": 12.56, "tpot_ms": 8.477342910999942, "tpot_relative": 1.12559, "ttft_error_pct": 18.97, "ttft_ms": 217.72314172506213, "ttft_relative": 1.189663, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7715.062197325012, "interactivity_tok_s": 121.25686684023006, "output_per_gpu": 466.57873117599536, "total_per_gpu": 4194.264532080802, "tpot_error_pct": 9.5, "tpot_ms": 8.246955624522409, "tpot_relative": 1.095, "ttft_error_pct": 26.03, "ttft_ms": 135.38105215000002, "ttft_relative": 0.739737, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 7105.605802250648, "interactivity_tok_s": 132.77627117330627, "output_per_gpu": 505.78347267599366, "total_per_gpu": 4546.691776992688, "tpot_ms": 7.5314662112686515, "tpot_relative": 1.0, "ttft_ms": 183.0124346750381, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12193.055708907943, "interactivity_tok_s": 85.58929253424249, "output_per_gpu": 671.2017229627659, "total_per_gpu": null, "tpot_error_pct": 16.96, "tpot_ms": 11.683704472728536, "tpot_relative": 1.551319, "ttft_error_pct": 1.87, "ttft_ms": 240.62603330665166, "ttft_relative": 1.314807, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10637.170996787541, "interactivity_tok_s": 88.37102785391497, "output_per_gpu": 686.4196745720639, "total_per_gpu": 6106.020364021281, "tpot_error_pct": 13.28, "tpot_ms": 11.31592586716415, "tpot_relative": 1.502486, "ttft_error_pct": 31.61, "ttft_ms": 161.55352241249975, "ttft_relative": 0.882746, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 9482.446413999787, "interactivity_tok_s": 100.10839885079051, "output_per_gpu": 769.0250610068946, "total_per_gpu": 6840.833468064921, "tpot_ms": 9.989171852508392, "tpot_relative": 1.326325, "ttft_ms": 236.21780332396156, "ttft_relative": 1.29072, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16241.133200023634, "interactivity_tok_s": 64.03098323986842, "output_per_gpu": 1007.811450002527, "total_per_gpu": null, "tpot_error_pct": 11.0, "tpot_ms": 15.617439392643238, "tpot_relative": 2.073625, "ttft_error_pct": 12.73, "ttft_ms": 264.4927013496021, "ttft_relative": 1.445217, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14086.631698968773, "interactivity_tok_s": 65.93501239819565, "output_per_gpu": 1014.0064475635203, "total_per_gpu": 9151.15438640948, "tpot_error_pct": 7.79, "tpot_ms": 15.166448956751324, "tpot_relative": 2.013745, "ttft_error_pct": 29.42, "ttft_ms": 213.90399976249986, "ttft_relative": 1.168795, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 13177.64753314982, "interactivity_tok_s": 71.07183127077764, "output_per_gpu": 1082.385483600832, "total_per_gpu": 9768.258071573275, "tpot_ms": 14.07027203492316, "tpot_relative": 1.868198, "ttft_ms": 303.0690713875174, "ttft_relative": 1.656003, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23105.926589514158, "interactivity_tok_s": 44.84021128689151, "output_per_gpu": 1416.7793649468324, "total_per_gpu": null, "tpot_error_pct": 5.03, "tpot_ms": 22.301411418468, "tpot_relative": 2.961098, "ttft_error_pct": 34.56, "ttft_ms": 291.58270842139433, "ttft_relative": 1.59324, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20276.031759712503, "interactivity_tok_s": 46.31530034752156, "output_per_gpu": 1429.6492526022905, "total_per_gpu": 12784.378827078974, "tpot_error_pct": 1.69, "tpot_ms": 21.591137107966794, "tpot_relative": 2.866791, "ttft_error_pct": 32.1, "ttft_ms": 302.5478726468745, "ttft_relative": 1.653155, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 20087.002899171875, "interactivity_tok_s": 47.096660987078394, "output_per_gpu": 1442.008635982088, "total_per_gpu": 12894.900368575132, "tpot_ms": 21.2329277498964, "tpot_relative": 2.819229, "ttft_ms": 445.5639902279927, "ttft_relative": 2.43461, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36372.24433585538, "interactivity_tok_s": 28.377451237915146, "output_per_gpu": 1800.0538926177394, "total_per_gpu": null, "tpot_error_pct": 8.46, "tpot_ms": 35.23924652767613, "tpot_relative": 4.678936, "ttft_error_pct": 52.38, "ttft_ms": 322.4951380427024, "ttft_relative": 1.762149, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32193.487994770287, "interactivity_tok_s": 29.063246698030355, "output_per_gpu": 1802.9130847645513, "total_per_gpu": 16251.058255946997, "tpot_error_pct": 5.9, "tpot_ms": 34.407718118697694, "tpot_relative": 4.568529, "ttft_error_pct": 27.92, "ttft_ms": 488.1067164359388, "ttft_relative": 2.667069, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 30617.837286711212, "interactivity_tok_s": 30.779369342958695, "output_per_gpu": 1894.5058879589142, "total_per_gpu": 17076.65544813303, "tpot_ms": 32.489294658948786, "tpot_relative": 4.313807, "ttft_ms": 677.1880429818793, "ttft_relative": 3.70023, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 51280.1908104374, "interactivity_tok_s": 19.53205689757102, "output_per_gpu": 2254.247235911095, "total_per_gpu": 20351.528545047608, "tpot_error_pct": 1.0, "tpot_ms": 51.197884853814784, "tpot_relative": 6.797864, "ttft_error_pct": 12.6, "ttft_ms": 4164.113737447658, "ttft_relative": 22.753174, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 51408.629482016615, "interactivity_tok_s": 19.72789258568024, "output_per_gpu": 2256.2326133726374, "total_per_gpu": 20369.452695263968, "tpot_ms": 50.68965150012341, "tpot_relative": 6.730383, "ttft_ms": 4764.5437532188, "ttft_relative": 26.033989, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 101236.86165941981, "interactivity_tok_s": 19.371501870405734, "output_per_gpu": 2259.753182512397, "total_per_gpu": 20331.931219853584, "tpot_error_pct": 1.26, "tpot_ms": 51.622223547247096, "tpot_relative": 6.854206, "ttft_error_pct": 1.02, "ttft_ms": 53622.6821964056, "ttft_relative": 293.000212, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 101199.28545249498, "interactivity_tok_s": 19.615238739390993, "output_per_gpu": 2265.6066050680233, "total_per_gpu": 20384.596876316748, "tpot_ms": 50.980771291445805, "tpot_relative": 6.769037, "ttft_ms": 54175.31930026248, "ttft_relative": 296.019882, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.33, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "193aa01c95a35bcb", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 107057.98085977847, "interactivity_tok_s": 12.107728105411846, "output_per_gpu": 2157.6749224464966, "total_per_gpu": 19419.231055120967, "tpot_error_pct": 17.94, "tpot_ms": 82.59187779027062, "tpot_relative": 1.179412, "ttft_error_pct": 35.33, "ttft_ms": 30791.94675190875, "ttft_relative": 0.646736, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26980953103", "measured": {"e2e_ms": 112247.25690557579, "interactivity_tok_s": 14.28000438656578, "output_per_gpu": 2046.0295877698331, "total_per_gpu": 18414.414932099913, "tpot_ms": 70.02798969311041, "tpot_relative": 1.0, "ttft_ms": 47611.28972389533, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 58.33, "points": 7, "status_counts": {"failed": 5, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.72, "tpot_shape_error_pct": 15.53, "ttft_mape_pct": 54.49, "ttft_shape_error_pct": 46.01}, "aisimulate": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.09, "tpot_shape_error_pct": 11.08, "ttft_mape_pct": 29.63, "ttft_shape_error_pct": 17.01}, "gpu": "gb200", "precisions": ["fp4"], "rows": 12, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.83, "tpot_shape_error_pct": 3.43, "ttft_mape_pct": 26.01, "ttft_shape_error_pct": 26.38}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.32, "tpot_shape_error_pct": 2.74, "ttft_mape_pct": 42.51, "ttft_shape_error_pct": 11.84}, "framework": "dynamo-vllm", "id": "5930abb3cfb8de8c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6996.354, "interactivity_tok_s": 151.6990291262136, "output_per_gpu": 111.73060000000001, "total_per_gpu": null, "tpot_error_pct": 9.33, "tpot_ms": 6.592, "tpot_relative": 1.093324, "ttft_error_pct": 10.22, "ttft_ms": 252.738, "ttft_relative": 0.89783, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5718.315549649998, "interactivity_tok_s": 165.06329785439027, "output_per_gpu": 125.3082819913564, "total_per_gpu": 1126.4467229520505, "tpot_error_pct": 0.48, "tpot_ms": 6.05828196212428, "tpot_relative": 1.004804, "ttft_error_pct": 46.72, "ttft_ms": 149.9910785500004, "ttft_relative": 0.53283, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 5823.112231091363, "interactivity_tok_s": 165.85624866935598, "output_per_gpu": 153.5818283828794, "total_per_gpu": 1105.555484628721, "tpot_ms": 6.029317604991524, "tpot_relative": 1.0, "ttft_ms": 281.4986912475433, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7814.754, "interactivity_tok_s": 135.28138528138527, "output_per_gpu": 199.3442, "total_per_gpu": null, "tpot_error_pct": 13.15, "tpot_ms": 7.392, "tpot_relative": 1.226009, "ttft_error_pct": 23.21, "ttft_ms": 252.738, "ttft_relative": 0.89783, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6480.247395062491, "interactivity_tok_s": 147.4578285681058, "output_per_gpu": 224.14707395989393, "total_per_gpu": 1993.891854845441, "tpot_error_pct": 3.81, "tpot_ms": 6.781599930709231, "tpot_relative": 1.124771, "ttft_error_pct": 36.9, "ttft_ms": 207.6793480625006, "ttft_relative": 0.737763, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 6371.230091273901, "interactivity_tok_s": 153.07188659390422, "output_per_gpu": 284.42079360112024, "total_per_gpu": 2026.0096175644182, "tpot_ms": 6.532878259042918, "tpot_relative": 1.083519, "ttft_ms": 329.1406451317016, "ttft_relative": 1.169244, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8613.717, "interactivity_tok_s": 122.3540927444023, "output_per_gpu": 360.6282, "total_per_gpu": null, "tpot_error_pct": 13.01, "tpot_ms": 8.173, "tpot_relative": 1.355543, "ttft_error_pct": 44.6, "ttft_ms": 252.738, "ttft_relative": 0.89783, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7043.352284768769, "interactivity_tok_s": 134.68529325295825, "output_per_gpu": 400.5550087695747, "total_per_gpu": 3614.9086963969835, "tpot_error_pct": 2.66, "tpot_ms": 7.424715615548737, "tpot_relative": 1.231435, "ttft_error_pct": 43.91, "ttft_ms": 255.85995188125202, "ttft_relative": 0.908921, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 7063.6456739433925, "interactivity_tok_s": 138.26809186147366, "output_per_gpu": 503.0708559281245, "total_per_gpu": 3634.459256262674, "tpot_ms": 7.232326609394941, "tpot_relative": 1.199527, "ttft_ms": 456.1858293847763, "ttft_relative": 1.620561, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 25.0, "points": 1, "status_counts": {"failed": 3, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.37, "tpot_shape_error_pct": null, "ttft_mape_pct": 79.93, "ttft_shape_error_pct": null}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.84, "tpot_shape_error_pct": 27.28, "ttft_mape_pct": 34.3, "ttft_shape_error_pct": 15.93}, "framework": "dynamo-vllm", "id": "1e7034f69b4f9a0f", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 23961.786, "interactivity_tok_s": 43.14808422506041, "output_per_gpu": 1695.43, "total_per_gpu": null, "tpot_error_pct": 10.37, "tpot_ms": 23.176, "tpot_relative": 1.103694, "ttft_error_pct": 79.93, "ttft_ms": 252.738, "ttft_relative": 0.200736, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19802.58675549108, "interactivity_tok_s": 48.746895423998716, "output_per_gpu": 1918.37277460856, "total_per_gpu": 17260.390917567365, "tpot_error_pct": 2.31, "tpot_ms": 20.51412692648499, "tpot_relative": 0.976929, "ttft_error_pct": 27.81, "ttft_ms": 908.8571862035182, "ttft_relative": 0.721855, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 20598.90141595888, "interactivity_tok_s": 47.62226899179296, "output_per_gpu": 2761.5838728926165, "total_per_gpu": 16571.598357385115, "tpot_ms": 20.998579470716447, "tpot_relative": 1.0, "ttft_ms": 1259.057227698304, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 29652.14878770516, "interactivity_tok_s": 33.87295334503515, "output_per_gpu": 2575.5332611490526, "total_per_gpu": 23179.986460468706, "tpot_error_pct": 4.7, "tpot_ms": 29.522078863742557, "tpot_relative": 1.405908, "ttft_error_pct": 54.43, "ttft_ms": 2459.9839162835915, "ttft_relative": 1.95383, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 31367.26348388606, "interactivity_tok_s": 35.46580504233026, "output_per_gpu": 3634.0121215048034, "total_per_gpu": 21816.698896477457, "tpot_ms": 28.19617371737223, "tpot_relative": 1.342766, "ttft_ms": 5398.533898167807, "ttft_relative": 4.287759, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 55770.04870096732, "interactivity_tok_s": 28.227991963002967, "output_per_gpu": 2715.3595455992977, "total_per_gpu": 24433.223998755257, "tpot_error_pct": 22.81, "tpot_ms": 35.425828422746136, "tpot_relative": 1.687058, "ttft_error_pct": 31.42, "ttft_ms": 23185.797683456265, "ttft_relative": 18.415206, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 60370.379020002336, "interactivity_tok_s": 34.66689150220974, "output_per_gpu": 3748.261581452773, "total_per_gpu": 22494.626668269997, "tpot_ms": 28.845966761578783, "tpot_relative": 1.37371, "ttft_ms": 33809.95948348546, "ttft_relative": 26.853394, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 109026.52988277069, "interactivity_tok_s": 23.61359301207544, "output_per_gpu": 2766.007459939355, "total_per_gpu": 24871.473982313128, "tpot_error_pct": 45.53, "tpot_ms": 42.34848968086404, "tpot_relative": 2.016731, "ttft_error_pct": 23.55, "ttft_ms": 70008.63533741266, "ttft_relative": 55.604014, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 118387.36415129782, "interactivity_tok_s": 34.36437062117844, "output_per_gpu": 3800.3984548958456, "total_per_gpu": 22793.46571091357, "tpot_ms": 29.099907314575095, "tpot_relative": 1.385804, "ttft_ms": 91572.8379281663, "ttft_relative": 72.731275, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 60.0, "points": 3, "status_counts": {"failed": 2, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.07, "tpot_shape_error_pct": 27.64, "ttft_mape_pct": 74.49, "ttft_shape_error_pct": 65.64}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.95, "tpot_shape_error_pct": 3.09, "ttft_mape_pct": 18.15, "ttft_shape_error_pct": 20.41}, "framework": "dynamo-vllm", "id": "059f24b87190054b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10391.691, "interactivity_tok_s": 100.89799212995662, "output_per_gpu": 594.6408, "total_per_gpu": null, "tpot_error_pct": 12.36, "tpot_ms": 9.911, "tpot_relative": 1.123644, "ttft_error_pct": 54.64, "ttft_ms": 252.738, "ttft_relative": 0.453579, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8613.881618203108, "interactivity_tok_s": 112.46683035738296, "output_per_gpu": 664.1313254827284, "total_per_gpu": 5938.8737765200885, "tpot_error_pct": 0.81, "tpot_ms": 8.89151047310861, "tpot_relative": 1.008061, "ttft_error_pct": 29.02, "ttft_ms": 395.5082031718745, "ttft_relative": 0.709804, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 8710.464186616446, "interactivity_tok_s": 113.37347309018875, "output_per_gpu": 823.0161507367025, "total_per_gpu": 5892.69014682978, "tpot_ms": 8.820405450616288, "tpot_relative": 1.0, "ttft_ms": 557.2079957230017, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13510.818, "interactivity_tok_s": 77.1604938271605, "output_per_gpu": 909.484, "total_per_gpu": null, "tpot_error_pct": 19.78, "tpot_ms": 12.96, "tpot_relative": 1.46932, "ttft_error_pct": 70.39, "ttft_ms": 252.738, "ttft_relative": 0.453579, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11217.695430871767, "interactivity_tok_s": 86.6215528601914, "output_per_gpu": 1017.2296483383982, "total_per_gpu": 9169.082200644538, "tpot_error_pct": 6.7, "tpot_ms": 11.544470942630367, "tpot_relative": 1.308837, "ttft_error_pct": 30.83, "ttft_ms": 590.339286793749, "ttft_relative": 1.059459, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 10812.556372879408, "interactivity_tok_s": 92.42284143997244, "output_per_gpu": 1316.7773697612986, "total_per_gpu": 9502.962314982298, "tpot_ms": 10.819836140284524, "tpot_relative": 1.226682, "ttft_ms": 853.4227831281896, "ttft_relative": 1.531605, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24135.696, "interactivity_tok_s": 42.833890173905594, "output_per_gpu": 1312.727, "total_per_gpu": null, "tpot_error_pct": 67.06, "tpot_ms": 23.346, "tpot_relative": 2.646817, "ttft_error_pct": 98.45, "ttft_ms": 252.738, "ttft_relative": 0.453579, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27332.171015031618, "interactivity_tok_s": 68.20690692200145, "output_per_gpu": 1659.7076990865655, "total_per_gpu": 14933.07456939686, "tpot_error_pct": 4.92, "tpot_ms": 14.661271785034291, "tpot_relative": 1.662199, "ttft_error_pct": 14.92, "ttft_ms": 13828.901249212253, "ttft_relative": 24.818203, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 29125.040379849725, "interactivity_tok_s": 71.55939349529203, "output_per_gpu": 1945.2084218337116, "total_per_gpu": 14007.27155359573, "tpot_ms": 13.974405751018434, "tpot_relative": 1.584327, "ttft_ms": 16254.55709009675, "ttft_relative": 29.171435, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 53393.06019845981, "interactivity_tok_s": 67.82159300414158, "output_per_gpu": 1689.8918139607563, "total_per_gpu": 15209.149094735869, "tpot_error_pct": 0.74, "tpot_ms": 14.744566674199678, "tpot_relative": 1.671643, "ttft_error_pct": 8.71, "ttft_ms": 39811.91051330553, "ttft_relative": 71.448922, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 57088.740678661496, "interactivity_tok_s": 68.32117996590536, "output_per_gpu": 1976.533774222506, "total_per_gpu": 14239.283998780214, "tpot_ms": 14.636749548222596, "tpot_relative": 1.659419, "ttft_ms": 43608.40413870424, "ttft_relative": 78.262345, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 105385.58537689903, "interactivity_tok_s": 67.69713737270288, "output_per_gpu": 1707.732603761855, "total_per_gpu": 15366.441363286158, "tpot_error_pct": 1.59, "tpot_ms": 14.771673350005255, "tpot_relative": 1.674716, "ttft_error_pct": 7.28, "ttft_ms": 91783.65889741393, "ttft_relative": 164.720642, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26843662304", "measured": {"e2e_ms": 112813.85237996912, "interactivity_tok_s": 66.62059308605767, "output_per_gpu": 1991.299555799378, "total_per_gpu": 14340.580811343763, "tpot_ms": 15.010373724956821, "tpot_relative": 1.701778, "ttft_ms": 98993.0290304309, "ttft_relative": 177.659025, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}]}, {"aic": {"coverage_pct": 84.62, "points": 11, "status_counts": {"failed": 2, "success": 11, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.04, "tpot_shape_error_pct": 10.83, "ttft_mape_pct": 53.59, "ttft_shape_error_pct": 42.85}, "aisimulate": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.17, "tpot_shape_error_pct": 8.07, "ttft_mape_pct": 41.2, "ttft_shape_error_pct": 13.34}, "gpu": "gb300", "precisions": ["fp4"], "rows": 13, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.87, "tpot_shape_error_pct": 4.03, "ttft_mape_pct": 62.11, "ttft_shape_error_pct": 30.12}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.74, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 38.12, "ttft_shape_error_pct": 8.78}, "framework": "dynamo-vllm", "id": "f8a79576769a86de", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 14274.627, "interactivity_tok_s": 72.82260413632392, "output_per_gpu": 715.3526666666667, "total_per_gpu": null, "tpot_error_pct": 26.42, "tpot_ms": 13.732, "tpot_relative": 1.264203, "ttft_error_pct": 55.4, "ttft_ms": 226.791, "ttft_relative": 0.446047, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11644.795532815657, "interactivity_tok_s": 81.15716713871026, "output_per_gpu": 821.7797452277913, "total_per_gpu": 7407.3401685906365, "tpot_error_pct": 13.44, "tpot_ms": 12.321770648929181, "tpot_relative": 1.134374, "ttft_error_pct": 40.72, "ttft_ms": 301.39093320156246, "ttft_relative": 0.592768, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 10507.473658018716, "interactivity_tok_s": 92.06258390256978, "output_per_gpu": 1367.8854525706656, "total_per_gpu": 8226.50067600769, "tpot_ms": 10.862176115524893, "tpot_relative": 1.0, "ttft_ms": 508.4463917761241, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18314.454, "interactivity_tok_s": 56.557886997341775, "output_per_gpu": 1111.1833333333334, "total_per_gpu": null, "tpot_error_pct": 21.32, "tpot_ms": 17.681, "tpot_relative": 1.627759, "ttft_error_pct": 68.83, "ttft_ms": 226.791, "ttft_relative": 0.446047, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14937.652259232742, "interactivity_tok_s": 63.51507367000875, "output_per_gpu": 1274.822321795856, "total_per_gpu": 11509.200259219242, "tpot_error_pct": 8.03, "tpot_ms": 15.744294105608367, "tpot_relative": 1.44946, "ttft_error_pct": 35.52, "ttft_ms": 469.1497107648438, "ttft_relative": 0.922712, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 14121.010585677459, "interactivity_tok_s": 68.61816755196918, "output_per_gpu": 2029.0248115266443, "total_per_gpu": 12217.682004147022, "tpot_ms": 14.573399956252581, "tpot_relative": 1.341665, "ttft_ms": 727.5673495865249, "ttft_relative": 1.430962, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.02, "tpot_shape_error_pct": 4.37, "ttft_mape_pct": 22.89, "ttft_shape_error_pct": 24.72}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.45, "tpot_shape_error_pct": 3.49, "ttft_mape_pct": 46.98, "ttft_shape_error_pct": 6.81}, "framework": "dynamo-vllm", "id": "95b1ce14bf4608f1", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6679.875, "interactivity_tok_s": 158.5288522511097, "output_per_gpu": 58.409000000000006, "total_per_gpu": null, "tpot_error_pct": 12.34, "tpot_ms": 6.308, "tpot_relative": 1.123381, "ttft_error_pct": 5.33, "ttft_ms": 226.791, "ttft_relative": 0.946664, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5562.718614000019, "interactivity_tok_s": 171.67668177824015, "output_per_gpu": 67.01994350664391, "total_per_gpu": 597.2460498892792, "tpot_error_pct": 3.73, "tpot_ms": 5.82490289095714, "tpot_relative": 1.037347, "ttft_error_pct": 49.56, "ttft_ms": 120.84315060000094, "ttft_relative": 0.50442, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5485.568011304713, "interactivity_tok_s": 178.08825390017185, "output_per_gpu": 84.91922379795987, "total_per_gpu": 605.9850337969872, "tpot_ms": 5.615193467844064, "tpot_relative": 1.0, "ttft_ms": 239.5686343050329, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7154.547, "interactivity_tok_s": 147.66686355581808, "output_per_gpu": 108.71600000000001, "total_per_gpu": null, "tpot_error_pct": 15.14, "tpot_ms": 6.772, "tpot_relative": 1.206014, "ttft_error_pct": 13.93, "ttft_ms": 226.791, "ttft_relative": 0.946664, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5857.029038374982, "interactivity_tok_s": 160.60685700102485, "output_per_gpu": 122.35152296892211, "total_per_gpu": 1099.8672227111188, "tpot_error_pct": 5.86, "tpot_ms": 6.226384219657688, "tpot_relative": 1.108846, "ttft_error_pct": 49.07, "ttft_ms": 134.20576322500065, "ttft_relative": 0.560198, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 5669.390624455991, "interactivity_tok_s": 170.0199761037337, "output_per_gpu": 157.72064205683984, "total_per_gpu": 1135.3486457421313, "tpot_ms": 5.881661807727072, "tpot_relative": 1.047455, "ttft_ms": 263.49039648193866, "ttft_relative": 1.099853, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7982.154, "interactivity_tok_s": 131.90871916633688, "output_per_gpu": 194.4454, "total_per_gpu": null, "tpot_error_pct": 18.87, "tpot_ms": 7.581, "tpot_relative": 1.350087, "ttft_error_pct": 28.62, "ttft_ms": 226.791, "ttft_relative": 0.946664, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6613.02131666244, "interactivity_tok_s": 143.64988458760394, "output_per_gpu": 219.73906667457126, "total_per_gpu": 1954.6805920480656, "tpot_error_pct": 9.15, "tpot_ms": 6.961370020385617, "tpot_relative": 1.239738, "ttft_error_pct": 45.18, "ttft_ms": 174.1681948000008, "ttft_relative": 0.727008, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 6216.38402190074, "interactivity_tok_s": 156.80001296106153, "output_per_gpu": 292.06119602370285, "total_per_gpu": 2080.434361248678, "tpot_ms": 6.377550493240916, "tpot_relative": 1.135767, "ttft_ms": 317.73589274889673, "ttft_relative": 1.326283, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8809.761, "interactivity_tok_s": 119.18951132300357, "output_per_gpu": 351.2074, "total_per_gpu": null, "tpot_error_pct": 17.73, "tpot_ms": 8.39, "tpot_relative": 1.49416, "ttft_error_pct": 43.66, "ttft_ms": 226.791, "ttft_relative": 0.946664, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7199.594530981243, "interactivity_tok_s": 131.07100463608967, "output_per_gpu": 392.04466164089155, "total_per_gpu": 3538.1049436756402, "tpot_error_pct": 7.05, "tpot_ms": 7.629452469494963, "tpot_relative": 1.358716, "ttft_error_pct": 44.11, "ttft_ms": 224.95495691874902, "ttft_relative": 0.939, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 6913.267099016593, "interactivity_tok_s": 140.31712908836278, "output_per_gpu": 513.9240783293162, "total_per_gpu": 3712.8688762028214, "tpot_ms": 7.1267136556810815, "tpot_relative": 1.269184, "ttft_ms": 402.51136542428867, "ttft_relative": 1.680151, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 66.44, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.73, "tpot_shape_error_pct": null, "ttft_mape_pct": 35.15, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ad026837f6ff4b15", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 24391.074, "interactivity_tok_s": 42.33521019431862, "output_per_gpu": 1247.5495, "total_per_gpu": null, "tpot_error_pct": 12.05, "tpot_ms": 23.621, "tpot_relative": 1.120475, "ttft_error_pct": 66.44, "ttft_ms": 226.791, "ttft_relative": 0.335594, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19712.259540632756, "interactivity_tok_s": 47.784622331328094, "output_per_gpu": 1451.8631152533299, "total_per_gpu": 13063.011141399782, "tpot_error_pct": 0.73, "tpot_ms": 20.927234562328845, "tpot_relative": 0.992695, "ttft_error_pct": 35.15, "ttft_ms": 438.2278304773433, "ttft_relative": 0.648467, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 20090.370890156057, "interactivity_tok_s": 47.435535577713516, "output_per_gpu": 2837.8775744762424, "total_per_gpu": 12772.063836248159, "tpot_ms": 21.08124189641967, "tpot_relative": 1.0, "ttft_ms": 675.7907412871873, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.15, "tpot_shape_error_pct": 19.55, "ttft_mape_pct": 76.81, "ttft_shape_error_pct": 65.23}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.58, "tpot_shape_error_pct": 9.73, "ttft_mape_pct": 35.29, "ttft_shape_error_pct": 7.33}, "framework": "dynamo-vllm", "id": "e621cabb1aa4d378", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10765.737, "interactivity_tok_s": 97.06853038245, "output_per_gpu": 572.0310000000001, "total_per_gpu": null, "tpot_error_pct": 17.87, "tpot_ms": 10.302, "tpot_relative": 1.178711, "ttft_error_pct": 54.6, "ttft_ms": 226.791, "ttft_relative": 0.45403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8893.320326109373, "interactivity_tok_s": 108.08663005090341, "output_per_gpu": 643.2538354217846, "total_per_gpu": 5752.180612856448, "tpot_error_pct": 5.86, "tpot_ms": 9.251838081444948, "tpot_relative": 1.058556, "ttft_error_pct": 31.52, "ttft_ms": 342.0661470156235, "ttft_relative": 0.684808, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 8578.960163803094, "interactivity_tok_s": 114.41569704551749, "output_per_gpu": 836.0059516368403, "total_per_gpu": 5985.695456269948, "tpot_ms": 8.740059500770899, "tpot_relative": 1.0, "ttft_ms": 499.50650320533896, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14274.627, "interactivity_tok_s": 72.82260413632392, "output_per_gpu": 858.4232, "total_per_gpu": null, "tpot_error_pct": 28.48, "tpot_ms": 13.732, "tpot_relative": 1.571156, "ttft_error_pct": 70.49, "ttft_ms": 226.791, "ttft_relative": 0.45403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11781.394116843812, "interactivity_tok_s": 81.66726041753951, "output_per_gpu": 972.5764074980774, "total_per_gpu": 8766.587801804646, "tpot_error_pct": 14.57, "tpot_ms": 12.24480893429397, "tpot_relative": 1.400998, "ttft_error_pct": 33.72, "ttft_ms": 509.3532459875023, "ttft_relative": 1.019713, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 10606.47085373721, "interactivity_tok_s": 93.56339261942594, "output_per_gpu": 1342.0527359633834, "total_per_gpu": 9685.370410710228, "tpot_ms": 10.687940785426123, "tpot_relative": 1.222868, "ttft_ms": 768.5275584047304, "ttft_relative": 1.538574, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18314.454, "interactivity_tok_s": 56.557886997341775, "output_per_gpu": 1333.42, "total_per_gpu": null, "tpot_error_pct": 26.84, "tpot_ms": 17.681, "tpot_relative": 2.022984, "ttft_error_pct": 84.02, "ttft_ms": 226.791, "ttft_relative": 0.45403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15298.868911754544, "interactivity_tok_s": 63.866076806643115, "output_per_gpu": 1487.7557636542256, "total_per_gpu": 13431.580799889784, "tpot_error_pct": 12.33, "tpot_ms": 15.657764653801056, "tpot_relative": 1.791494, "ttft_error_pct": 35.89, "ttft_ms": 909.9821411492123, "ttft_relative": 1.821762, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 14230.446928205765, "interactivity_tok_s": 71.73856135720811, "output_per_gpu": 2003.8036479341336, "total_per_gpu": 14478.976676951808, "tpot_ms": 13.939504515858575, "tpot_relative": 1.594898, "ttft_ms": 1419.3120017944693, "ttft_relative": 2.841428, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25949.103, "interactivity_tok_s": 39.77091950365893, "output_per_gpu": 1462.9478, "total_per_gpu": null, "tpot_error_pct": 67.42, "tpot_ms": 25.144, "tpot_relative": 2.876868, "ttft_error_pct": 98.13, "ttft_ms": 226.791, "ttft_relative": 0.45403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24086.0326467164, "interactivity_tok_s": 54.77270445248058, "output_per_gpu": 1892.8693210181893, "total_per_gpu": 17030.92582895462, "tpot_error_pct": 21.56, "tpot_ms": 18.25726901740948, "tpot_relative": 2.088918, "ttft_error_pct": 40.01, "ttft_ms": 7270.582967499625, "ttft_relative": 14.555532, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 25952.678966224175, "interactivity_tok_s": 66.58378903065085, "output_per_gpu": 2184.818433928936, "total_per_gpu": 15732.681781469544, "tpot_ms": 15.018670678829421, "tpot_relative": 1.718372, "ttft_ms": 12120.317503896558, "ttft_relative": 24.264584, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.93, "tpot_shape_error_pct": 20.16, "ttft_mape_pct": 47.55, "ttft_shape_error_pct": 55.47}, "framework": "dynamo-vllm", "id": "c50c28a2bf383aea", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 31293.96288932558, "interactivity_tok_s": 30.965111548988048, "output_per_gpu": 2443.172772963424, "total_per_gpu": 21984.045437452503, "tpot_error_pct": 4.4, "tpot_ms": 32.29440973974726, "tpot_relative": 1.044004, "ttft_error_pct": 27.42, "ttft_ms": 1557.0788818361193, "ttft_relative": 0.725838, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 30627.167185373462, "interactivity_tok_s": 32.327713970297665, "output_per_gpu": 3730.2353998226704, "total_per_gpu": 22386.445257444764, "tpot_ms": 30.93321108070891, "tpot_relative": 1.0, "ttft_ms": 2145.2165549473475, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 52543.980812168564, "interactivity_tok_s": 19.364214602769117, "output_per_gpu": 2916.54755391579, "total_per_gpu": 26225.10519439669, "tpot_error_pct": 25.45, "tpot_ms": 51.64165035936951, "tpot_relative": 1.669457, "ttft_error_pct": 67.68, "ttft_ms": 4959.620787925776, "ttft_relative": 2.311944, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26883183237", "measured": {"e2e_ms": 53275.96871622194, "interactivity_tok_s": 24.2929003487119, "output_per_gpu": 4291.744945635611, "total_per_gpu": 25740.390756214238, "tpot_ms": 41.16429021012404, "tpot_relative": 1.330747, "ttft_ms": 15344.945225370277, "ttft_relative": 7.153098, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 100}]}, {"aic": {"coverage_pct": 86.62, "points": 136, "status_counts": {"failed": 21, "success": 136, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.9, "tpot_shape_error_pct": 21.38, "ttft_mape_pct": 45.88, "ttft_shape_error_pct": 47.29}, "aisimulate": {"coverage_pct": 100.0, "points": 157, "status_counts": {"failed": 0, "success": 157, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.16, "tpot_shape_error_pct": 18.31, "ttft_mape_pct": 31.96, "ttft_shape_error_pct": 38.37}, "frameworks": ["dynamo-vllm", "vllm"], "gpu_skus": ["b200", "b300"], "hf_model_paths": ["nvidia/MiniMax-M3-NVFP4"], "model": "nvidia/MiniMax-M3-NVFP4", "precisions": ["fp4"], "rows": 157, "workloads": [{"aic": {"coverage_pct": 98.21, "points": 55, "status_counts": {"failed": 1, "success": 55, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.97, "tpot_shape_error_pct": 20.0, "ttft_mape_pct": 49.77, "ttft_shape_error_pct": 49.65}, "aisimulate": {"coverage_pct": 100.0, "points": 56, "status_counts": {"failed": 0, "success": 56, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.72, "tpot_shape_error_pct": 18.78, "ttft_mape_pct": 35.98, "ttft_shape_error_pct": 42.5}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.47, "tpot_shape_error_pct": 23.52, "ttft_mape_pct": 45.72, "ttft_shape_error_pct": 56.68}, "aisimulate": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.45, "tpot_shape_error_pct": 20.54, "ttft_mape_pct": 28.62, "ttft_shape_error_pct": 19.45}, "gpu": "b200", "precisions": ["fp4"], "rows": 17, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.9, "tpot_shape_error_pct": 31.18, "ttft_mape_pct": 45.47, "ttft_shape_error_pct": 61.06}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.49, "tpot_shape_error_pct": 26.2, "ttft_mape_pct": 13.53, "ttft_shape_error_pct": 7.73}, "framework": "vllm", "id": "e53d1e88db8cb35f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 25584.962942458464, "interactivity_tok_s": 40.244766822857954, "output_per_gpu": 639.7507800504634, "total_per_gpu": null, "tpot_error_pct": 61.71, "tpot_ms": 24.847951148570868, "tpot_relative": 1.617141, "ttft_error_pct": 7.72, "ttft_ms": 165.5089174704668, "ttft_relative": 1.077158, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23860.323193354674, "interactivity_tok_s": 38.8833412519584, "output_per_gpu": 603.496317528344, "total_per_gpu": 1207.264737823587, "tpot_error_pct": 67.38, "tpot_ms": 25.717954471045722, "tpot_relative": 1.673762, "ttft_error_pct": 9.74, "ttft_ms": 168.6116159140626, "ttft_relative": 1.097351, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 14309.996357106258, "interactivity_tok_s": 65.08145408084081, "output_per_gpu": 1003.464181041481, "total_per_gpu": 2007.380801264801, "tpot_ms": 15.36536044136094, "tpot_relative": 1.0, "ttft_ms": 153.6532856250176, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30251.817963967977, "interactivity_tok_s": 34.01829178792746, "output_per_gpu": 1082.1167851462963, "total_per_gpu": null, "tpot_error_pct": 17.49, "tpot_ms": 29.39594986820837, "tpot_relative": 1.913131, "ttft_error_pct": 35.49, "ttft_ms": 179.76124879081664, "ttft_relative": 1.169915, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29208.114495911817, "interactivity_tok_s": 31.797573473423338, "output_per_gpu": 981.8428984089473, "total_per_gpu": 1965.8427477530452, "tpot_error_pct": 25.7, "tpot_ms": 31.44894061918932, "tpot_relative": 2.046743, "ttft_error_pct": 4.62, "ttft_ms": 291.5248273640626, "ttft_relative": 1.89729, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 23325.768069082023, "interactivity_tok_s": 39.96808594332003, "output_per_gpu": 1235.8325337481838, "total_per_gpu": 2474.379992810465, "tpot_ms": 25.019962212304357, "tpot_relative": 1.628336, "ttft_ms": 278.66395775699715, "ttft_relative": 1.813589, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35783.59861014621, "interactivity_tok_s": 28.746031192058446, "output_per_gpu": 1829.6650572599435, "total_per_gpu": null, "tpot_error_pct": 11.43, "tpot_ms": 34.787410940967256, "tpot_relative": 2.264015, "ttft_error_pct": 61.14, "ttft_ms": 196.07721753670592, "ttft_relative": 1.276102, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 36250.41873789244, "interactivity_tok_s": 25.83186245745859, "output_per_gpu": 1587.1538226420503, "total_per_gpu": 3172.7736662625266, "tpot_error_pct": 24.0, "tpot_ms": 38.711881562812515, "tpot_relative": 2.519426, "ttft_error_pct": 10.61, "ttft_ms": 558.1274854312502, "ttft_relative": 3.632382, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 29278.04690657735, "interactivity_tok_s": 32.03172710284776, "output_per_gpu": 1975.359588760123, "total_per_gpu": 3948.8099988848826, "tpot_ms": 31.219047189968588, "tpot_relative": 2.031781, "ttft_ms": 504.6021758613162, "ttft_relative": 3.284031, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 47044.20974703098, "interactivity_tok_s": 21.841904307281176, "output_per_gpu": 2783.4243726086615, "total_per_gpu": null, "tpot_error_pct": 4.98, "tpot_ms": 45.78355375664941, "tpot_relative": 2.97966, "ttft_error_pct": 77.52, "ttft_ms": 207.6342539786345, "ttft_relative": 1.351317, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 49805.22024004426, "interactivity_tok_s": 18.966912959950953, "output_per_gpu": 2317.418246495925, "total_per_gpu": 4632.701735853346, "tpot_error_pct": 20.89, "tpot_ms": 52.72339268448807, "tpot_relative": 3.431315, "ttft_error_pct": 29.18, "ttft_ms": 1193.1988202119132, "ttft_relative": 7.765528, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 41148.248411952736, "interactivity_tok_s": 22.929326064066736, "output_per_gpu": 2814.842292522258, "total_per_gpu": 5627.091611295901, "tpot_ms": 43.612271778328946, "tpot_relative": 2.83835, "ttft_ms": 923.7027559212904, "ttft_relative": 6.011604, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.26, "tpot_shape_error_pct": 32.73, "ttft_mape_pct": 46.43, "ttft_shape_error_pct": 55.31}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.21, "tpot_shape_error_pct": 27.52, "ttft_mape_pct": 31.25, "ttft_shape_error_pct": 21.05}, "framework": "vllm", "id": "048da5b35f2e8cad", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 24174.311689226306, "interactivity_tok_s": 42.7136994455786, "output_per_gpu": 1354.1647191795478, "total_per_gpu": null, "tpot_error_pct": 21.02, "tpot_ms": 23.411692571234603, "tpot_relative": 0.78979, "ttft_error_pct": 15.15, "ttft_ms": 224.15018885330636, "ttft_relative": 0.848549, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33307.85840447563, "interactivity_tok_s": 27.88884393700854, "output_per_gpu": 864.2018489362943, "total_per_gpu": 1727.5684495886717, "tpot_error_pct": 20.96, "tpot_ms": 35.8566314996298, "tpot_relative": 1.209619, "ttft_error_pct": 20.03, "ttft_ms": 211.24550901445417, "ttft_relative": 0.799696, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 27619.458802713656, "interactivity_tok_s": 33.73487342093232, "output_per_gpu": 1041.8190256785033, "total_per_gpu": 2082.6311366478785, "tpot_ms": 29.642915434195913, "tpot_relative": 1.0, "ttft_ms": 264.15715156753237, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29263.320468350594, "interactivity_tok_s": 35.25460838262423, "output_per_gpu": 2237.34008827913, "total_per_gpu": null, "tpot_error_pct": 50.42, "tpot_ms": 28.365086037740966, "tpot_relative": 0.956893, "ttft_error_pct": 56.83, "ttft_ms": 245.83745174158605, "ttft_relative": 0.930648, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44568.407275535574, "interactivity_tok_s": 20.86471557432071, "output_per_gpu": 1297.7978306361783, "total_per_gpu": 2594.4001570997825, "tpot_error_pct": 16.23, "tpot_ms": 47.92780406893023, "tpot_relative": 1.616838, "ttft_error_pct": 42.24, "ttft_ms": 328.89475513203104, "ttft_relative": 1.245072, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 53484.61068883595, "interactivity_tok_s": 17.477485838342524, "output_per_gpu": 1087.2941207735796, "total_per_gpu": 2173.5866489820346, "tpot_ms": 57.21646747418191, "tpot_relative": 1.93019, "ttft_ms": 569.4568406613367, "ttft_relative": 2.15575, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 42551.76572568745, "interactivity_tok_s": 24.19702435057562, "output_per_gpu": 3077.287105878014, "total_per_gpu": null, "tpot_error_pct": 43.32, "tpot_ms": 41.32739569591792, "tpot_relative": 1.394174, "ttft_error_pct": 67.33, "ttft_ms": 273.8399287634196, "ttft_relative": 1.036655, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 62169.766007072, "interactivity_tok_s": 14.975503669075742, "output_per_gpu": 1867.5031324316424, "total_per_gpu": 3734.8962793642722, "tpot_error_pct": 8.42, "tpot_ms": 66.77571733797438, "tpot_relative": 2.25267, "ttft_error_pct": 31.49, "ttft_ms": 574.1680961175749, "ttft_relative": 2.173585, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 68168.211009612, "interactivity_tok_s": 13.714710571531702, "output_per_gpu": 1708.4970310356905, "total_per_gpu": 3416.8934411432247, "tpot_ms": 72.91440783852552, "tpot_relative": 2.459758, "ttft_ms": 838.0791109944325, "ttft_relative": 3.172653, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.36, "tpot_shape_error_pct": 18.93, "ttft_mape_pct": 45.61, "ttft_shape_error_pct": 55.52}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.61, "tpot_shape_error_pct": 17.1, "ttft_mape_pct": 33.87, "ttft_shape_error_pct": 23.0}, "framework": "vllm", "id": "55673b49feef8414", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6977.851838451626, "interactivity_tok_s": 148.2983171268052, "output_per_gpu": 18.325840525208868, "total_per_gpu": null, "tpot_error_pct": 20.01, "tpot_ms": 6.743164854291177, "tpot_relative": 1.200116, "ttft_error_pct": 92.8, "ttft_ms": 79.5941925117523, "ttft_relative": 1.927968, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6250.775852399996, "interactivity_tok_s": 147.95462372853353, "output_per_gpu": 18.425712698652976, "total_per_gpu": 36.72144153303284, "tpot_error_pct": 20.29, "tpot_ms": 6.75882898958802, "tpot_relative": 1.202904, "ttft_error_pct": 26.9, "ttft_ms": 30.17823840000035, "ttft_relative": 0.730991, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 5212.842241999897, "interactivity_tok_s": 177.97518485871763, "output_per_gpu": 22.09229845284698, "total_per_gpu": 44.02874717708151, "tpot_ms": 5.618760844630288, "tpot_relative": 1.0, "ttft_ms": 41.28398980010388, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7152.169293133313, "interactivity_tok_s": 144.91111454823266, "output_per_gpu": 35.758381760557825, "total_per_gpu": null, "tpot_error_pct": 14.55, "tpot_ms": 6.900781924958261, "tpot_relative": 1.228168, "ttft_error_pct": 55.4, "ttft_ms": 92.66938390101132, "ttft_relative": 2.244681, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6457.537454749997, "interactivity_tok_s": 145.52330409458642, "output_per_gpu": 36.09338300266978, "total_per_gpu": 71.52683887425627, "tpot_error_pct": 14.07, "tpot_ms": 6.87175161546652, "tpot_relative": 1.223001, "ttft_error_pct": 37.41, "ttft_ms": 37.32500774999916, "ttft_relative": 0.904104, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 5688.139639950021, "interactivity_tok_s": 165.99371478824068, "output_per_gpu": 40.973088528673756, "total_per_gpu": 81.19702996957385, "tpot_ms": 6.024324482861937, "tpot_relative": 1.07218, "ttft_ms": 59.6325255000238, "ttft_relative": 1.444447, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9678.443415560065, "interactivity_tok_s": 106.83706005633647, "output_per_gpu": 52.84940749642239, "total_per_gpu": null, "tpot_error_pct": 40.2, "tpot_ms": 9.360047903533548, "tpot_relative": 1.665856, "ttft_error_pct": 8.54, "ttft_ms": 103.11441024524498, "ttft_relative": 2.497685, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8554.183576774318, "interactivity_tok_s": 107.68849364283471, "output_per_gpu": 52.25776660577557, "total_per_gpu": 105.02659421052887, "tpot_error_pct": 39.09, "tpot_ms": 9.286043161831683, "tpot_relative": 1.652685, "ttft_error_pct": 63.62, "ttft_ms": 41.01737274999912, "ttft_relative": 0.993542, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 6226.0578457000065, "interactivity_tok_s": 149.78158172153792, "output_per_gpu": 71.3986367059384, "total_per_gpu": 143.49552481010153, "tpot_ms": 6.676388301594524, "tpot_relative": 1.188231, "ttft_ms": 112.74131152495102, "ttft_relative": 2.730872, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11019.460552375245, "interactivity_tok_s": 93.80181435722554, "output_per_gpu": 92.83576043833583, "total_per_gpu": null, "tpot_error_pct": 39.31, "tpot_ms": 10.660774600709738, "tpot_relative": 1.897353, "ttft_error_pct": 19.46, "ttft_ms": 113.48813584918364, "ttft_relative": 2.748962, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9912.747650199988, "interactivity_tok_s": 94.10998847018627, "output_per_gpu": 91.9920185444405, "total_per_gpu": 183.30454918181084, "tpot_error_pct": 38.86, "tpot_ms": 10.625864653216876, "tpot_relative": 1.89114, "ttft_error_pct": 49.25, "ttft_ms": 48.21423117499948, "ttft_relative": 1.167868, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 7199.200038999999, "interactivity_tok_s": 130.67664338731143, "output_per_gpu": 126.34621570362602, "total_per_gpu": 251.75919038229048, "tpot_ms": 7.652476939096976, "tpot_relative": 1.361951, "ttft_ms": 94.99947498750885, "ttft_relative": 2.301121, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14294.965667861414, "interactivity_tok_s": 72.18926470454369, "output_per_gpu": 143.12731121837595, "total_per_gpu": null, "tpot_error_pct": 53.91, "tpot_ms": 13.852475213493326, "tpot_relative": 2.465397, "ttft_error_pct": 42.69, "ttft_ms": 123.88352445773961, "ttft_relative": 3.000764, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12766.36336240625, "interactivity_tok_s": 72.0231309271162, "output_per_gpu": 140.7246932139058, "total_per_gpu": 282.9474485762164, "tpot_error_pct": 54.26, "tpot_ms": 13.884428337501044, "tpot_relative": 2.471084, "ttft_error_pct": 30.53, "ttft_ms": 60.31615533125023, "ttft_relative": 1.461006, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 8324.070897862526, "interactivity_tok_s": 111.10593325538073, "output_per_gpu": 215.26450955331782, "total_per_gpu": 432.8205829131851, "tpot_ms": 9.0004194258597, "tpot_relative": 1.601851, "ttft_ms": 86.82099902501932, "ttft_relative": 2.103019, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17264.173546294776, "interactivity_tok_s": 59.72180854661198, "output_per_gpu": 237.02264050040333, "total_per_gpu": null, "tpot_error_pct": 53.04, "tpot_ms": 16.74430202862184, "tpot_relative": 2.98007, "ttft_error_pct": 17.75, "ttft_ms": 134.75257101463816, "ttft_relative": 3.264039, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15755.553490518772, "interactivity_tok_s": 58.88341652877893, "output_per_gpu": 229.3053833274042, "total_per_gpu": 457.8854137619935, "tpot_error_pct": 55.21, "tpot_ms": 16.98271022557354, "tpot_relative": 3.022501, "ttft_error_pct": 49.03, "ttft_ms": 83.5051796124998, "ttft_relative": 2.022701, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 10262.086100518723, "interactivity_tok_s": 91.39564163397819, "output_per_gpu": 351.4438472976877, "total_per_gpu": 701.7759858007528, "tpot_ms": 10.94144077465757, "tpot_relative": 1.947305, "ttft_ms": 163.83991413430863, "ttft_relative": 3.968607, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18906.80030096853, "interactivity_tok_s": 54.52986074178205, "output_per_gpu": 432.8601280873931, "total_per_gpu": null, "tpot_error_pct": 33.43, "tpot_ms": 18.33857608284293, "tpot_relative": 3.263811, "ttft_error_pct": 29.94, "ttft_ms": 146.43696822021303, "ttft_relative": 3.547064, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17552.7724407235, "interactivity_tok_s": 52.86896778634086, "output_per_gpu": 408.5260469953177, "total_per_gpu": 817.2362890959063, "tpot_error_pct": 37.62, "tpot_ms": 18.914687421954138, "tpot_relative": 3.366345, "ttft_error_pct": 38.8, "ttft_ms": 127.91642469218763, "ttft_relative": 3.098451, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 12873.326074596956, "interactivity_tok_s": 72.7586915806646, "output_per_gpu": 557.7054987986927, "total_per_gpu": 1115.6624543253392, "tpot_ms": 13.744062438112163, "tpot_relative": 2.446102, "ttft_ms": 209.00684192039023, "ttft_relative": 5.062661, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23058.773248961126, "interactivity_tok_s": 44.67357724819853, "output_per_gpu": 709.8382825173682, "total_per_gpu": null, "tpot_error_pct": 4.2, "tpot_ms": 22.384596479573958, "tpot_relative": 3.983903, "ttft_error_pct": 46.92, "ttft_ms": 159.33105035696659, "ttft_relative": 3.859391, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22058.294675010133, "interactivity_tok_s": 42.104954395228425, "output_per_gpu": 648.9880513270209, "total_per_gpu": 1299.4018250241681, "tpot_error_pct": 1.64, "tpot_ms": 23.750174162718622, "tpot_relative": 4.226942, "ttft_error_pct": 26.53, "ttft_ms": 220.54058184531178, "ttft_relative": 5.342037, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 21796.267975068735, "interactivity_tok_s": 42.79579293859553, "output_per_gpu": 662.1306344113883, "total_per_gpu": 1325.7158633341792, "tpot_ms": 23.3667828385567, "tpot_relative": 4.158707, "ttft_ms": 300.18762037186235, "ttft_relative": 7.271284, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26945.769628448183, "interactivity_tok_s": 38.211929555900674, "output_per_gpu": 1214.884579338151, "total_per_gpu": null, "tpot_error_pct": 8.87, "tpot_ms": 26.169837839177642, "tpot_relative": 4.657582, "ttft_error_pct": 63.96, "ttft_ms": 174.02551896945576, "ttft_relative": 4.215327, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26733.691587293182, "interactivity_tok_s": 35.037594359722206, "output_per_gpu": 1074.3946430736469, "total_per_gpu": 2147.750887209613, "tpot_error_pct": 0.61, "tpot_ms": 28.54077222691862, "tpot_relative": 5.079549, "ttft_error_pct": 13.35, "ttft_ms": 418.460734624611, "ttft_relative": 10.136151, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 26934.257868125325, "interactivity_tok_s": 34.82418268190263, "output_per_gpu": 1074.010956208926, "total_per_gpu": 2146.983884312282, "tpot_ms": 28.715677525999148, "tpot_relative": 5.110678, "ttft_ms": 482.90861902142694, "ttft_relative": 11.697237, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36636.848855734825, "interactivity_tok_s": 28.064074616428535, "output_per_gpu": 1787.0532549840614, "total_per_gpu": null, "tpot_error_pct": 6.06, "tpot_ms": 35.632744484459366, "tpot_relative": 6.341744, "ttft_error_pct": 78.65, "ttft_ms": 184.55124813289206, "ttft_relative": 4.470286, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37434.22753061595, "interactivity_tok_s": 25.232205600193446, "output_per_gpu": 1540.8412389802143, "total_per_gpu": 3080.2630872918744, "tpot_error_pct": 4.49, "tpot_ms": 39.63189012665359, "tpot_relative": 7.053493, "ttft_error_pct": 3.26, "ttft_ms": 892.7183033933594, "ttft_relative": 21.623838, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 35843.99005807403, "interactivity_tok_s": 26.364241076741, "output_per_gpu": 1616.3171196062049, "total_per_gpu": 3231.1453217438507, "tpot_ms": 37.93016446364609, "tpot_relative": 6.750628, "ttft_ms": 864.5679493187444, "ttft_relative": 20.941967, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 97.44, "points": 38, "status_counts": {"failed": 1, "success": 38, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.3, "tpot_shape_error_pct": 18.11, "ttft_mape_pct": 51.58, "ttft_shape_error_pct": 45.86}, "aisimulate": {"coverage_pct": 100.0, "points": 39, "status_counts": {"failed": 0, "success": 39, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.09, "tpot_shape_error_pct": 17.87, "ttft_mape_pct": 39.18, "ttft_shape_error_pct": 54.45}, "gpu": "b300", "precisions": ["fp4"], "rows": 39, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.72, "tpot_shape_error_pct": null, "ttft_mape_pct": 74.93, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.0, "tpot_shape_error_pct": null, "ttft_mape_pct": 85.78, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ede1320a72b9677a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7232.686, "interactivity_tok_s": 143.92630972941853, "output_per_gpu": 24.065863636363634, "total_per_gpu": null, "tpot_error_pct": 8.72, "tpot_ms": 6.948, "tpot_relative": 1.087228, "ttft_error_pct": 74.93, "ttft_ms": 124.882, "ttft_relative": 0.25066, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5988.208166550003, "interactivity_tok_s": 154.92602712957918, "output_per_gpu": 26.94899867353411, "total_per_gpu": 54.161548261661345, "tpot_error_pct": 1.0, "tpot_ms": 6.454693369007688, "tpot_relative": 1.010035, "ttft_error_pct": 85.78, "ttft_ms": 70.84694645000238, "ttft_relative": 0.142202, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6345.074820617447, "interactivity_tok_s": 156.48067803674917, "output_per_gpu": 35.00124369014587, "total_per_gpu": 51.50004351455207, "tpot_ms": 6.39056535635123, "tpot_relative": 1.0, "ttft_ms": 498.2131593627855, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 22}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.38, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.67, "tpot_shape_error_pct": null, "ttft_mape_pct": 55.07, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "b838c08354bffc48", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10039.798, "interactivity_tok_s": 103.17787866281469, "output_per_gpu": 76.02585, "total_per_gpu": null, "tpot_error_pct": 34.56, "tpot_ms": 9.692, "tpot_relative": 1.345617, "ttft_error_pct": 33.38, "ttft_ms": 124.882, "ttft_relative": 0.666179, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8234.770875100015, "interactivity_tok_s": 112.26483635659373, "output_per_gpu": 86.89543116982298, "total_per_gpu": 174.71589371354253, "tpot_error_pct": 23.67, "tpot_ms": 8.907508641651944, "tpot_relative": 1.2367, "ttft_error_pct": 55.07, "ttft_ms": 84.23003362500074, "ttft_relative": 0.449323, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6814.479810635385, "interactivity_tok_s": 138.83790321905602, "output_per_gpu": 130.649852160818, "total_per_gpu": 210.73915840194832, "tpot_ms": 7.202644067753007, "tpot_relative": 1.0, "ttft_ms": 187.46001492982032, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.79, "tpot_shape_error_pct": 40.3, "ttft_mape_pct": 86.99, "ttft_shape_error_pct": 93.78}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.91, "tpot_shape_error_pct": 6.27, "ttft_mape_pct": 202.03, "ttft_shape_error_pct": 637.37}, "framework": "dynamo-vllm", "id": "cc2037dbae60fe9e", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 22999.162, "interactivity_tok_s": 44.72271914132379, "output_per_gpu": 1757.21475, "total_per_gpu": null, "tpot_error_pct": 2.2, "tpot_ms": 22.36, "tpot_relative": 1.021952, "ttft_error_pct": 75.5, "ttft_ms": 124.882, "ttft_relative": 0.245019, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19279.165112666546, "interactivity_tok_s": 48.59958976510442, "output_per_gpu": 1963.0155688131051, "total_per_gpu": 3924.222848809612, "tpot_error_pct": 5.96, "tpot_ms": 20.576305372808356, "tpot_relative": 0.940429, "ttft_error_pct": 36.61, "ttft_ms": 323.10827720957144, "ttft_relative": 0.633939, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 20671.083659849955, "interactivity_tok_s": 45.704484935867036, "output_per_gpu": 2731.771120004765, "total_per_gpu": 3675.2151140495303, "tpot_ms": 21.879690831287334, "tpot_relative": 1.0, "ttft_ms": 509.6832999511207, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71077.093, "interactivity_tok_s": 14.418155341205646, "output_per_gpu": 4427.827166666667, "total_per_gpu": null, "tpot_error_pct": 43.38, "tpot_ms": 69.357, "tpot_relative": 3.169926, "ttft_error_pct": 98.48, "ttft_ms": 124.882, "ttft_relative": 0.245019, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 77539.4834259971, "interactivity_tok_s": 23.451896325833353, "output_per_gpu": 3899.202510786736, "total_per_gpu": 7801.238542585921, "tpot_error_pct": 11.85, "tpot_ms": 42.6404750433104, "tpot_relative": 1.948861, "ttft_error_pct": 367.45, "ttft_ms": 38293.42886256071, "ttft_relative": 75.13181, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 52715.05524952617, "interactivity_tok_s": 20.672038946751787, "output_per_gpu": 8682.307392307972, "total_per_gpu": 11688.939184072444, "tpot_ms": 48.37452186384985, "tpot_relative": 2.210933, "ttft_ms": 8192.016753032942, "ttft_relative": 16.072759, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.3, "tpot_shape_error_pct": null, "ttft_mape_pct": 98.49, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.86, "tpot_shape_error_pct": null, "ttft_mape_pct": 75.95, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "6ade292fca108210", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 44355.31, "interactivity_tok_s": 23.128874086409475, "output_per_gpu": 2656.6963, "total_per_gpu": null, "tpot_error_pct": 33.3, "tpot_ms": 43.236, "tpot_relative": 1.333022, "ttft_error_pct": 98.49, "ttft_ms": 124.882, "ttft_relative": 0.015062, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37811.49384180053, "interactivity_tok_s": 25.723424399370362, "output_per_gpu": 4821.947694194967, "total_per_gpu": 9641.781172403225, "tpot_error_pct": 19.86, "tpot_ms": 38.875072948082185, "tpot_relative": 1.198569, "ttft_error_pct": 75.95, "ttft_ms": 1994.0339817113702, "ttft_relative": 0.240494, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 38180.09900024885, "interactivity_tok_s": 30.831303444066286, "output_per_gpu": 5942.257391936625, "total_per_gpu": 9593.680227439036, "tpot_ms": 32.43456773775996, "tpot_relative": 1.0, "ttft_ms": 8291.40740347392, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.65, "tpot_shape_error_pct": null, "ttft_mape_pct": 25.68, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.28, "tpot_shape_error_pct": null, "ttft_mape_pct": 41.09, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e2a723dc67752cc4", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 14535.883, "interactivity_tok_s": 70.98743522396536, "output_per_gpu": 174.36333333333334, "total_per_gpu": null, "tpot_error_pct": 44.65, "tpot_ms": 14.087, "tpot_relative": 1.446509, "ttft_error_pct": 25.68, "ttft_ms": 124.882, "ttft_relative": 0.743158, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12074.220914087477, "interactivity_tok_s": 77.04419136298317, "output_per_gpu": 198.99162179924758, "total_per_gpu": 397.35378106070584, "tpot_error_pct": 33.28, "tpot_ms": 12.979563836144854, "tpot_relative": 1.332793, "ttft_error_pct": 41.09, "ttft_ms": 98.99222054999947, "ttft_relative": 0.589091, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 9176.255756385217, "interactivity_tok_s": 102.68399076433559, "output_per_gpu": 393.6943296653547, "total_per_gpu": 528.1125628355888, "tpot_ms": 9.738616434328554, "tpot_relative": 1.0, "ttft_ms": 168.04229095287155, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.48, "tpot_shape_error_pct": 27.33, "ttft_mape_pct": 20.12, "ttft_shape_error_pct": 38.98}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.99, "tpot_shape_error_pct": 26.6, "ttft_mape_pct": 35.53, "ttft_shape_error_pct": 20.87}, "framework": "dynamo-vllm", "id": "7f401fd413130aee", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7206.088, "interactivity_tok_s": 144.4669170759896, "output_per_gpu": 22.138833333333334, "total_per_gpu": null, "tpot_error_pct": 6.63, "tpot_ms": 6.922, "tpot_relative": 1.066287, "ttft_error_pct": 43.88, "ttft_ms": 124.882, "ttft_relative": 1.438778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5959.169076600003, "interactivity_tok_s": 156.19284843718862, "output_per_gpu": 25.769811980948855, "total_per_gpu": 51.35783127915821, "tpot_error_pct": 1.38, "tpot_ms": 6.4023417845673, "tpot_relative": 0.986237, "ttft_error_pct": 23.57, "ttft_ms": 66.34129189999928, "ttft_relative": 0.764325, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6078.654089919291, "interactivity_tok_s": 154.04313418494186, "output_per_gpu": 37.999370680397064, "total_per_gpu": 50.51503642036911, "tpot_ms": 6.491688222854614, "tpot_relative": 1.0, "ttft_ms": 86.7972407140769, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9520.114, "interactivity_tok_s": 108.8850174216028, "output_per_gpu": 66.88766666666668, "total_per_gpu": null, "tpot_error_pct": 28.3, "tpot_ms": 9.184, "tpot_relative": 1.414732, "ttft_error_pct": 9.16, "ttft_ms": 124.882, "ttft_relative": 1.438778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7820.1995939500075, "interactivity_tok_s": 118.35040097335447, "output_per_gpu": 76.09204521104765, "total_per_gpu": 152.92824156298082, "tpot_error_pct": 18.04, "tpot_ms": 8.449485525825477, "tpot_relative": 1.301585, "ttft_error_pct": 45.92, "ttft_ms": 74.34388122499918, "ttft_relative": 0.856524, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6687.047793829697, "interactivity_tok_s": 139.70076586248132, "output_per_gpu": 134.70245114092748, "total_per_gpu": 181.68164661889872, "tpot_ms": 7.158156892170371, "tpot_relative": 1.102665, "ttft_ms": 137.46895978110842, "ttft_relative": 1.583794, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11612.149, "interactivity_tok_s": 89.0551251224508, "output_per_gpu": 109.28133333333334, "total_per_gpu": null, "tpot_error_pct": 40.19, "tpot_ms": 11.229, "tpot_relative": 1.72975, "ttft_error_pct": 6.26, "ttft_ms": 124.882, "ttft_relative": 1.438778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9670.83319871252, "interactivity_tok_s": 96.83183327208626, "output_per_gpu": 125.76340738414548, "total_per_gpu": 250.59787858641792, "tpot_error_pct": 28.93, "tpot_ms": 10.327182355311972, "tpot_relative": 1.590832, "ttft_error_pct": 36.59, "ttft_ms": 84.46840712500156, "ttft_relative": 0.973169, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 7483.851256695925, "interactivity_tok_s": 124.84752684089999, "output_per_gpu": 238.9337029814299, "total_per_gpu": 322.31689258634765, "tpot_ms": 8.009770199728141, "tpot_relative": 1.23385, "ttft_ms": 133.21825813909527, "ttft_relative": 1.534821, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13680.655, "interactivity_tok_s": 75.4660025658441, "output_per_gpu": 185.27566666666667, "total_per_gpu": null, "tpot_error_pct": 38.82, "tpot_ms": 13.251, "tpot_relative": 2.041226, "ttft_error_pct": 21.21, "ttft_ms": 124.882, "ttft_relative": 1.438778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11246.364450981355, "interactivity_tok_s": 82.09796247160496, "output_per_gpu": 212.51455808481774, "total_per_gpu": 427.291405808938, "tpot_error_pct": 27.6, "tpot_ms": 12.18057025892534, "tpot_relative": 1.876333, "ttft_error_pct": 36.05, "ttft_ms": 101.35746598749861, "ttft_relative": 1.16775, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 8939.767108381056, "interactivity_tok_s": 104.75906479329724, "output_per_gpu": 400.615224215808, "total_per_gpu": 538.4960500524675, "tpot_ms": 9.545713318204255, "tpot_relative": 1.470452, "ttft_ms": 158.49274672291358, "ttft_relative": 1.826011, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.02, "tpot_shape_error_pct": 8.78, "ttft_mape_pct": 42.09, "ttft_shape_error_pct": 30.15}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.0, "tpot_shape_error_pct": 30.39, "ttft_mape_pct": 50.1, "ttft_shape_error_pct": 107.81}, "framework": "dynamo-vllm", "id": "96b44f672552a98a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 8487.907, "interactivity_tok_s": 122.32415902140671, "output_per_gpu": 45.031400000000005, "total_per_gpu": null, "tpot_error_pct": 25.24, "tpot_ms": 8.175, "tpot_relative": 1.252432, "ttft_error_pct": 27.5, "ttft_ms": 124.882, "ttft_relative": 0.724952, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6970.699333849998, "interactivity_tok_s": 132.90435921893325, "output_per_gpu": 51.104091220245245, "total_per_gpu": 102.70796093481253, "tpot_error_pct": 15.27, "tpot_ms": 7.524207677437433, "tpot_relative": 1.152729, "ttft_error_pct": 57.57, "ttft_ms": 73.0851191249981, "ttft_relative": 0.424266, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6145.2978349872865, "interactivity_tok_s": 153.20265748868803, "output_per_gpu": 73.36019822203843, "total_per_gpu": 118.73460204810624, "tpot_ms": 6.527301917552158, "tpot_relative": 1.0, "ttft_ms": 172.2623989917338, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12664.816, "interactivity_tok_s": 81.57937673356176, "output_per_gpu": 120.2094, "total_per_gpu": null, "tpot_error_pct": 47.18, "tpot_ms": 12.258, "tpot_relative": 1.877958, "ttft_error_pct": 23.88, "ttft_ms": 124.882, "ttft_relative": 0.724952, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10382.916323643773, "interactivity_tok_s": 88.97578346879183, "output_per_gpu": 138.24838090336448, "total_per_gpu": 277.96846276964936, "tpot_error_pct": 34.95, "tpot_ms": 11.239013145085135, "tpot_relative": 1.721847, "ttft_error_pct": 39.61, "ttft_ms": 99.07547911874978, "ttft_relative": 0.575143, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 7825.997960503446, "interactivity_tok_s": 120.06862196754591, "output_per_gpu": 228.66437402259032, "total_per_gpu": 368.83729251153943, "tpot_ms": 8.328570642463909, "tpot_relative": 1.275959, "ttft_ms": 164.05743041686947, "ttft_relative": 0.952369, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16589.044, "interactivity_tok_s": 62.13495712687958, "output_per_gpu": 366.18649999999997, "total_per_gpu": null, "tpot_error_pct": 33.36, "tpot_ms": 16.094, "tpot_relative": 2.465644, "ttft_error_pct": 52.25, "ttft_ms": 124.882, "ttft_relative": 0.724952, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13764.906643118607, "interactivity_tok_s": 67.65006727989999, "output_per_gpu": 415.7255400890126, "total_per_gpu": 831.6385213710255, "tpot_error_pct": 22.49, "tpot_ms": 14.781951300396083, "tpot_relative": 2.264634, "ttft_error_pct": 41.0, "ttft_ms": 154.29068000156033, "ttft_relative": 0.895672, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 11350.463104625305, "interactivity_tok_s": 82.86213422500393, "output_per_gpu": 628.7520126142355, "total_per_gpu": 1017.6407280701463, "tpot_ms": 12.068238518749698, "tpot_relative": 1.848886, "ttft_ms": 261.52568003999477, "ttft_relative": 1.518182, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18446.812, "interactivity_tok_s": 55.83472920156337, "output_per_gpu": 658.1371, "total_per_gpu": null, "tpot_error_pct": 22.32, "tpot_ms": 17.91, "tpot_relative": 2.74386, "ttft_error_pct": 64.71, "ttft_ms": 124.882, "ttft_relative": 0.724952, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15323.793031560193, "interactivity_tok_s": 60.81598561506222, "output_per_gpu": 742.7215239350642, "total_per_gpu": 1487.074687604147, "tpot_error_pct": 12.3, "tpot_ms": 16.443045194228198, "tpot_relative": 2.519118, "ttft_error_pct": 39.5, "ttft_ms": 214.10993824062547, "ttft_relative": 1.242929, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 13804.824041037864, "interactivity_tok_s": 68.29588048274125, "output_per_gpu": 1031.42965303222, "total_per_gpu": 1669.0078631769775, "tpot_ms": 14.6421715765522, "tpot_relative": 2.24322, "ttft_ms": 353.9197079673613, "ttft_relative": 2.054538, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 257419.79986206992, "interactivity_tok_s": 54.63911640495807, "output_per_gpu": 1393.840625514013, "total_per_gpu": 2788.694144482385, "tpot_error_pct": 94.99, "tpot_ms": 18.301906505743894, "tpot_relative": 2.803901, "ttft_error_pct": 72.79, "ttft_ms": 240574.02817901332, "ttft_relative": 1396.555659, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 474672.4370173646, "interactivity_tok_s": 2.74011937529533, "output_per_gpu": 976.0079793163494, "total_per_gpu": 1576.7925365331598, "tpot_ms": 364.9476037489133, "tpot_relative": 55.910943, "ttft_ms": 139230.41169138928, "ttft_relative": 808.246097, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.57, "tpot_shape_error_pct": 26.67, "ttft_mape_pct": 51.07, "ttft_shape_error_pct": 49.24}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.44, "tpot_shape_error_pct": 23.04, "ttft_mape_pct": 15.91, "ttft_shape_error_pct": 37.15}, "framework": "vllm", "id": "08ba3e3637290323", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 21211.75416202292, "interactivity_tok_s": 48.57943800386423, "output_per_gpu": 771.6476381432387, "total_per_gpu": null, "tpot_error_pct": 31.63, "tpot_ms": 20.584840852223433, "tpot_relative": 1.31628, "ttft_error_pct": 22.43, "ttft_ms": 153.46197019835097, "ttft_relative": 0.775723, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19726.4836349781, "interactivity_tok_s": 47.04159808501154, "output_per_gpu": 729.4825949159176, "total_per_gpu": 1459.294096946454, "tpot_error_pct": 35.93, "tpot_ms": 21.25778121297757, "tpot_relative": 1.35931, "ttft_error_pct": 27.55, "ttft_ms": 143.3224041078127, "ttft_relative": 0.724469, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28671299958", "measured": {"e2e_ms": 14603.69561967018, "interactivity_tok_s": 63.94411910960767, "output_per_gpu": 984.4738278219814, "total_per_gpu": 1969.39153250735, "tpot_ms": 15.638654717971539, "tpot_relative": 1.0, "ttft_ms": 197.8309604790411, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25119.873093432703, "interactivity_tok_s": 40.998367066862045, "output_per_gpu": 1303.191297115209, "total_per_gpu": null, "tpot_error_pct": 0.11, "tpot_ms": 24.391215346922316, "tpot_relative": 1.559675, "ttft_error_pct": 45.21, "ttft_ms": 167.65979353117177, "ttft_relative": 0.84749, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24063.47352005542, "interactivity_tok_s": 38.608712046955176, "output_per_gpu": 1191.5615082458876, "total_per_gpu": 2385.740685483082, "tpot_error_pct": 6.3, "tpot_ms": 25.90088990235725, "tpot_relative": 1.65621, "ttft_error_pct": 18.99, "ttft_ms": 247.87849397265637, "ttft_relative": 1.252981, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28671299958", "measured": {"e2e_ms": 22773.80061261356, "interactivity_tok_s": 41.042760751179344, "output_per_gpu": 1262.3025969764017, "total_per_gpu": 2527.378269738556, "tpot_ms": 24.364832718307465, "tpot_relative": 1.557988, "ttft_ms": 305.99146915774327, "ttft_relative": 1.546732, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30312.48703671277, "interactivity_tok_s": 33.954713874041666, "output_per_gpu": 2159.901954620352, "total_per_gpu": null, "tpot_error_pct": 3.63, "tpot_ms": 29.450991803659363, "tpot_relative": 1.883218, "ttft_error_pct": 57.68, "ttft_ms": 184.1224215692415, "ttft_relative": 0.930706, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30052.03188789832, "interactivity_tok_s": 31.17357851227099, "output_per_gpu": 1914.9468632775854, "total_per_gpu": 3828.042936623035, "tpot_error_pct": 4.97, "tpot_ms": 32.07844744569077, "tpot_relative": 2.051228, "ttft_error_pct": 9.14, "ttft_ms": 474.8093738605441, "ttft_relative": 2.400076, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28671299958", "measured": {"e2e_ms": 28586.66821456533, "interactivity_tok_s": 32.72350591152724, "output_per_gpu": 2020.7623474264533, "total_per_gpu": 4039.5716345985584, "tpot_ms": 30.55907281767563, "tpot_relative": 1.954073, "ttft_ms": 435.0640113583495, "ttft_relative": 2.19917, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 42291.52657348483, "interactivity_tok_s": 24.303193476313197, "output_per_gpu": 3096.2230642697323, "total_per_gpu": null, "tpot_error_pct": 6.9, "tpot_ms": 41.146855904950826, "tpot_relative": 2.631099, "ttft_error_pct": 78.98, "ttft_ms": 198.2929827201403, "ttft_relative": 1.002335, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42809.448265840656, "interactivity_tok_s": 22.06236687802041, "output_per_gpu": 2698.3708416049017, "total_per_gpu": 5394.256000521685, "tpot_error_pct": 2.55, "tpot_ms": 45.326052527766095, "tpot_relative": 2.898335, "ttft_error_pct": 7.95, "ttft_ms": 1018.1873468751979, "ttft_relative": 5.146754, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28671299958", "measured": {"e2e_ms": 41692.31499633183, "interactivity_tok_s": 22.62532703905376, "output_per_gpu": 2775.4619185935508, "total_per_gpu": 5548.367139813927, "tpot_ms": 44.19825615222674, "tpot_relative": 2.826219, "ttft_ms": 943.231821577092, "ttft_relative": 4.767868, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.72, "tpot_shape_error_pct": 14.1, "ttft_mape_pct": 71.63, "ttft_shape_error_pct": 46.75}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.22, "tpot_shape_error_pct": 12.78, "ttft_mape_pct": 22.54, "ttft_shape_error_pct": 15.79}, "framework": "vllm", "id": "3825836dd9e50931", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6680.146057984273, "interactivity_tok_s": 154.8724780757422, "output_per_gpu": 19.142545520716673, "total_per_gpu": null, "tpot_error_pct": 25.86, "tpot_ms": 6.456925158199756, "tpot_relative": 1.258561, "ttft_error_pct": 128.68, "ttft_ms": 74.71162114592252, "ttft_relative": 2.286763, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5970.139626499996, "interactivity_tok_s": 154.8395869946226, "output_per_gpu": 19.29184360927946, "total_per_gpu": 38.44759324909906, "tpot_error_pct": 25.88, "tpot_ms": 6.458296740579196, "tpot_relative": 1.258828, "ttft_error_pct": 20.56, "ttft_ms": 25.955024200000413, "ttft_relative": 0.794428, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 4754.77466431912, "interactivity_tok_s": 194.91644093803552, "output_per_gpu": 24.221115642831144, "total_per_gpu": 48.27136385384288, "tpot_ms": 5.130403547220025, "tpot_relative": 1.0, "ttft_ms": 32.67134800553322, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6713.906715890936, "interactivity_tok_s": 154.37352199863255, "output_per_gpu": 38.09257572713564, "total_per_gpu": null, "tpot_error_pct": 16.17, "tpot_ms": 6.477794812564152, "tpot_relative": 1.262629, "ttft_error_pct": 115.28, "ttft_ms": 87.12262263780747, "ttft_relative": 2.666637, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6067.732061900009, "interactivity_tok_s": 154.81381317379962, "output_per_gpu": 38.40923971667236, "total_per_gpu": 76.1162094529597, "tpot_error_pct": 15.84, "tpot_ms": 6.459371935224951, "tpot_relative": 1.259038, "ttft_error_pct": 18.93, "ttft_ms": 32.81047179999997, "ttft_relative": 1.004258, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 5250.224653421901, "interactivity_tok_s": 179.34050971041094, "output_per_gpu": 44.40182897713713, "total_per_gpu": 87.9918201830804, "tpot_ms": 5.575985044398191, "tpot_relative": 1.086851, "ttft_ms": 40.4695067438297, "ttft_relative": 1.238685, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7928.159794740904, "interactivity_tok_s": 130.62913909818207, "output_per_gpu": 64.51686308584502, "total_per_gpu": null, "tpot_error_pct": 23.51, "tpot_ms": 7.655259821075532, "tpot_relative": 1.492136, "ttft_error_pct": 52.24, "ttft_ms": 96.82899778063432, "ttft_relative": 2.963728, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7017.202941050005, "interactivity_tok_s": 131.31086077881957, "output_per_gpu": 63.4720406955248, "total_per_gpu": 127.56481370763935, "tpot_error_pct": 22.87, "tpot_ms": 7.615516295216456, "tpot_relative": 1.484389, "ttft_error_pct": 44.12, "ttft_ms": 35.53975802499983, "ttft_relative": 1.087796, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 5742.413488193415, "interactivity_tok_s": 161.33599668319104, "output_per_gpu": 77.48538311066842, "total_per_gpu": 155.7285436747333, "tpot_ms": 6.19824478453906, "tpot_relative": 1.20814, "ttft_ms": 63.60240115900524, "ttft_relative": 1.946733, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9450.8944288203, "interactivity_tok_s": 109.4778240091279, "output_per_gpu": 108.24372314226508, "total_per_gpu": null, "tpot_error_pct": 34.5, "tpot_ms": 9.134269967921753, "tpot_relative": 1.780419, "ttft_error_pct": 118.0, "ttft_ms": 106.53625163634729, "ttft_relative": 3.260847, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8504.959530874987, "interactivity_tok_s": 109.6915066996356, "output_per_gpu": 107.13912470587636, "total_per_gpu": 213.48687923895278, "tpot_error_pct": 34.24, "tpot_ms": 9.116476107291195, "tpot_relative": 1.776951, "ttft_error_pct": 14.68, "ttft_ms": 41.69699933749979, "ttft_relative": 1.276256, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 6352.659652545117, "interactivity_tok_s": 147.2509335195763, "output_per_gpu": 142.98030805060515, "total_per_gpu": 284.90450936709595, "tpot_ms": 6.79112842342052, "tpot_relative": 1.323703, "ttft_ms": 48.86883941362612, "ttft_relative": 1.495771, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12020.02779900962, "interactivity_tok_s": 85.93997585568088, "output_per_gpu": 170.2159124930292, "total_per_gpu": null, "tpot_error_pct": 41.75, "tpot_ms": 11.636028402885536, "tpot_relative": 2.268053, "ttft_error_pct": 81.37, "ttft_ms": 116.37074285771669, "ttft_relative": 3.561859, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10723.448903237499, "interactivity_tok_s": 85.75628542314146, "output_per_gpu": 167.4765623241153, "total_per_gpu": 336.7359695280653, "tpot_error_pct": 42.05, "tpot_ms": 11.660952839383928, "tpot_relative": 2.272911, "ttft_error_pct": 18.63, "ttft_ms": 52.210171099999414, "ttft_relative": 1.598042, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 7576.633381674765, "interactivity_tok_s": 121.8174459221065, "output_per_gpu": 236.4974941244957, "total_per_gpu": 475.5125843868773, "tpot_ms": 8.209004813969159, "tpot_relative": 1.60007, "ttft_ms": 64.16033759596758, "ttft_relative": 1.963811, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14010.727295178907, "interactivity_tok_s": 73.68124127004629, "output_per_gpu": 292.061926107723, "total_per_gpu": null, "tpot_error_pct": 37.76, "tpot_ms": 13.571975481994643, "tpot_relative": 2.645401, "ttft_error_pct": 24.82, "ttft_ms": 126.59637709838768, "ttft_relative": 3.874844, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12792.24847227499, "interactivity_tok_s": 72.54999757806084, "output_per_gpu": 282.0961737653849, "total_per_gpu": 563.3000035625491, "tpot_error_pct": 39.9, "tpot_ms": 13.783597979090775, "tpot_relative": 2.68665, "ttft_error_pct": 28.86, "ttft_ms": 72.15089118437456, "ttft_relative": 2.208384, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 9193.28231892432, "interactivity_tok_s": 101.49968811289905, "output_per_gpu": 391.693697444042, "total_per_gpu": 782.148365291763, "tpot_ms": 9.852247022549376, "tpot_relative": 1.920365, "ttft_ms": 101.42357440272463, "ttft_relative": 3.104358, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16402.511226268853, "interactivity_tok_s": 62.89679666758233, "output_per_gpu": 498.94798955499016, "total_per_gpu": null, "tpot_error_pct": 24.97, "tpot_ms": 15.899060889938939, "tpot_relative": 3.098988, "ttft_error_pct": 14.69, "ttft_ms": 137.77193586131747, "ttft_relative": 4.216904, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15189.073866624927, "interactivity_tok_s": 61.1011350487382, "output_per_gpu": 472.20343665768183, "total_per_gpu": 944.6197791076974, "tpot_error_pct": 28.64, "tpot_ms": 16.36630807598477, "tpot_relative": 3.190063, "ttft_error_pct": 30.75, "ttft_ms": 111.83360912031185, "ttft_relative": 3.422987, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 11881.055837674285, "interactivity_tok_s": 78.60195532860068, "output_per_gpu": 603.6353092379462, "total_per_gpu": 1207.5427839109927, "tpot_ms": 12.72232981761629, "tpot_relative": 2.479791, "ttft_ms": 161.4971863800747, "ttft_relative": 4.943083, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19119.9356400983, "interactivity_tok_s": 53.9284850116955, "output_per_gpu": 856.0698272264607, "total_per_gpu": null, "tpot_error_pct": 8.37, "tpot_ms": 18.5430760716369, "tpot_relative": 3.61435, "ttft_error_pct": 48.97, "ttft_ms": 150.36881881375402, "ttft_relative": 4.602468, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18207.311952058593, "interactivity_tok_s": 51.04301849867501, "output_per_gpu": 785.6697036049538, "total_per_gpu": 1573.06539718718, "tpot_error_pct": 3.19, "tpot_ms": 19.59131786114801, "tpot_relative": 3.81867, "ttft_error_pct": 34.63, "ttft_ms": 192.60588544765702, "ttft_relative": 5.895254, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 18898.25282177353, "interactivity_tok_s": 49.41529876050303, "output_per_gpu": 762.1113799726735, "total_per_gpu": 1525.8969960745533, "tpot_ms": 20.236647861760705, "tpot_relative": 3.944455, "ttft_ms": 294.64974456968775, "ttft_relative": 9.018598, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23230.88389330109, "interactivity_tok_s": 44.351047086502376, "output_per_gpu": 1409.1586075827202, "total_per_gpu": null, "tpot_error_pct": 15.57, "tpot_ms": 22.54738198287851, "tpot_relative": 4.394855, "ttft_error_pct": 56.93, "ttft_ms": 164.9121248163754, "ttft_relative": 5.047607, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22831.94275682188, "interactivity_tok_s": 41.044745982298224, "output_per_gpu": 1258.2454569598722, "total_per_gpu": 2515.274823766772, "tpot_error_pct": 8.77, "tpot_ms": 24.363654252636376, "tpot_relative": 4.748877, "ttft_error_pct": 4.03, "ttft_ms": 367.44114548554643, "ttft_relative": 11.246587, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 25011.313756455183, "interactivity_tok_s": 37.443945286057755, "output_per_gpu": 1155.768645938344, "total_per_gpu": 2310.4202452291797, "tpot_ms": 26.706587469893293, "tpot_relative": 5.205553, "ttft_ms": 382.87410506000015, "ttft_relative": 11.718956, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32560.605536311043, "interactivity_tok_s": 31.588799621030603, "output_per_gpu": 2010.773415346552, "total_per_gpu": null, "tpot_error_pct": 8.79, "tpot_ms": 31.65679012805028, "tpot_relative": 6.170429, "ttft_error_pct": 75.3, "ttft_ms": 175.70923531560862, "ttft_relative": 5.378083, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32525.21377261171, "interactivity_tok_s": 29.048731981448103, "output_per_gpu": 1774.2112057749832, "total_per_gpu": 3546.7880453571274, "tpot_error_pct": 0.81, "tpot_ms": 34.42491054820043, "tpot_relative": 6.709981, "ttft_error_pct": 10.25, "ttft_ms": 784.2986535033197, "ttft_relative": 24.005702, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 32728.459601061466, "interactivity_tok_s": 28.812517328648667, "output_per_gpu": 1766.6442639738718, "total_per_gpu": 3531.6611322631634, "tpot_ms": 34.70713747756038, "tpot_relative": 6.764992, "ttft_ms": 711.3821148040188, "ttft_relative": 21.773883, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.48, "tpot_shape_error_pct": 17.9, "ttft_mape_pct": 42.08, "ttft_shape_error_pct": 48.07}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.62, "tpot_shape_error_pct": 13.02, "ttft_mape_pct": 27.68, "ttft_shape_error_pct": 8.72}, "framework": "vllm", "id": "775369dcde8714d7", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 20117.565517690695, "interactivity_tok_s": 51.38858999302591, "output_per_gpu": 1627.234665706101, "total_per_gpu": null, "tpot_error_pct": 31.28, "tpot_ms": 19.459572643182327, "tpot_relative": 0.687167, "ttft_error_pct": 14.77, "ttft_ms": 210.4227037151772, "ttft_relative": 0.852345, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28511.578045949118, "interactivity_tok_s": 32.59481342093536, "output_per_gpu": 1010.3923400318511, "total_per_gpu": 2019.8081391443172, "tpot_error_pct": 8.34, "tpot_ms": 30.67972769427509, "tpot_relative": 1.083379, "ttft_error_pct": 23.22, "ttft_ms": 189.54849172109385, "ttft_relative": 0.767792, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 26378.173687691105, "interactivity_tok_s": 35.3125271148004, "output_per_gpu": 1090.9102197288314, "total_per_gpu": 2180.7660782687144, "tpot_ms": 28.318562326310367, "tpot_relative": 1.0, "ttft_ms": 246.8749370362275, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25454.104016639663, "interactivity_tok_s": 40.557073789208815, "output_per_gpu": 2572.158892617086, "total_per_gpu": null, "tpot_error_pct": 46.33, "tpot_ms": 24.656611204185893, "tpot_relative": 0.870687, "ttft_error_pct": 47.19, "ttft_ms": 230.39075475749695, "ttft_relative": 0.933229, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 39264.71542530005, "interactivity_tok_s": 23.688065377421005, "output_per_gpu": 1474.6750212771929, "total_per_gpu": 2947.9916028193898, "tpot_error_pct": 8.11, "tpot_ms": 42.21535123561337, "tpot_relative": 1.490731, "ttft_error_pct": 32.06, "ttft_ms": 296.38777430371084, "ttft_relative": 1.200558, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 42871.418202615816, "interactivity_tok_s": 21.767272219353288, "output_per_gpu": 1352.5949406738232, "total_per_gpu": 2703.943899225311, "tpot_ms": 45.94052897040998, "tpot_relative": 1.622276, "ttft_ms": 436.25820750926323, "ttft_relative": 1.767122, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37182.180085257714, "interactivity_tok_s": 27.70397589878811, "output_per_gpu": 3521.686993601478, "total_per_gpu": null, "tpot_error_pct": 40.84, "tpot_ms": 36.09590203418219, "tpot_relative": 1.274638, "ttft_error_pct": 64.28, "ttft_ms": 256.07230428933633, "ttft_relative": 1.037255, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 54868.36662538246, "interactivity_tok_s": 16.97251901673543, "output_per_gpu": 2117.5346347177574, "total_per_gpu": 4234.944558478177, "tpot_error_pct": 3.43, "tpot_ms": 58.91877328368105, "tpot_relative": 2.080571, "ttft_error_pct": 27.76, "ttft_ms": 517.882485211329, "ttft_relative": 2.097752, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 57056.40202255145, "interactivity_tok_s": 16.39104924569252, "output_per_gpu": 2037.5058461369117, "total_per_gpu": 4074.8916945649376, "tpot_ms": 61.008907057172955, "tpot_relative": 2.154379, "ttft_ms": 716.9299010942879, "ttft_relative": 2.904021, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.61, "tpot_shape_error_pct": 10.34, "ttft_mape_pct": 38.67, "ttft_shape_error_pct": 41.05}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 37.16, "tpot_shape_error_pct": 10.31, "ttft_mape_pct": 8.44, "ttft_shape_error_pct": 11.48}, "framework": "vllm", "id": "4db33b06e64b0542", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 18323.492178043354, "interactivity_tok_s": 56.29768732563901, "output_per_gpu": 446.63975188128774, "total_per_gpu": null, "tpot_error_pct": 41.89, "tpot_ms": 17.762718994401418, "tpot_relative": 1.418879, "ttft_error_pct": 63.59, "ttft_ms": 152.23064677070155, "ttft_relative": 1.635915, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16378.554174037516, "interactivity_tok_s": 56.186750350940336, "output_per_gpu": 438.6471156584245, "total_per_gpu": 881.9637788245918, "tpot_error_pct": 42.17, "tpot_ms": 17.79779029315697, "tpot_relative": 1.421681, "ttft_error_pct": 1.95, "ttft_ms": 91.23876440000035, "ttft_relative": 0.980479, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 11549.212489407273, "interactivity_tok_s": 79.87960775236832, "output_per_gpu": 621.5492490428929, "total_per_gpu": 1249.7151009157192, "tpot_ms": 12.518839640525794, "tpot_relative": 1.0, "ttft_ms": 93.05532951257192, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24109.069900983315, "interactivity_tok_s": 42.7265242641902, "output_per_gpu": 678.9146187399131, "total_per_gpu": null, "tpot_error_pct": 45.61, "tpot_ms": 23.404665303845377, "tpot_relative": 1.869555, "ttft_error_pct": 38.64, "ttft_ms": 166.09729514949444, "ttft_relative": 1.78493, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22009.923192881146, "interactivity_tok_s": 42.171453336010046, "output_per_gpu": 657.9610014286704, "total_per_gpu": 1313.8407001473026, "tpot_error_pct": 47.53, "tpot_ms": 23.712723202406302, "tpot_relative": 1.894163, "ttft_error_pct": 5.95, "ttft_ms": 126.9263175062501, "ttft_relative": 1.363988, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 14951.785769221897, "interactivity_tok_s": 62.215337377695064, "output_per_gpu": 966.5318187369384, "total_per_gpu": 1930.0062445747424, "tpot_ms": 16.0732070603303, "tpot_relative": 1.283921, "ttft_ms": 119.80369079101365, "ttft_relative": 1.287446, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30776.402799442647, "interactivity_tok_s": 33.43584415815792, "output_per_gpu": 1063.6720676333507, "total_per_gpu": null, "tpot_error_pct": 43.62, "tpot_ms": 29.908023116443815, "tpot_relative": 2.389041, "ttft_error_pct": 1.54, "ttft_ms": 180.49515132062407, "ttft_relative": 1.939654, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28736.97005718441, "interactivity_tok_s": 32.27919078158743, "output_per_gpu": 1004.21264210441, "total_per_gpu": 2008.8780608547333, "tpot_error_pct": 48.76, "tpot_ms": 30.979710946484325, "tpot_relative": 2.474647, "ttft_error_pct": 11.69, "ttft_ms": 198.54015899375003, "ttft_relative": 2.133571, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 19360.87303691529, "interactivity_tok_s": 48.01943705433345, "output_per_gpu": 1486.4659381349047, "total_per_gpu": 2973.602089961128, "tpot_ms": 20.82490052660366, "tpot_relative": 1.663485, "ttft_ms": 177.76093025677255, "ttft_relative": 1.910271, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36274.04488520362, "interactivity_tok_s": 28.35560592287055, "output_per_gpu": 1804.9269169511995, "total_per_gpu": null, "tpot_error_pct": 3.33, "tpot_ms": 35.26639503737207, "tpot_relative": 2.817066, "ttft_error_pct": 50.89, "ttft_ms": 196.5227619719933, "ttft_relative": 2.111892, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34918.08967817486, "interactivity_tok_s": 26.593545634445917, "output_per_gpu": 1644.4880574918711, "total_per_gpu": 3292.588790758246, "tpot_error_pct": 10.17, "tpot_ms": 37.603109180925706, "tpot_relative": 3.003722, "ttft_error_pct": 14.19, "ttft_ms": 343.39752800468693, "ttft_relative": 3.690251, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 31902.41763210397, "interactivity_tok_s": 29.298484475092152, "output_per_gpu": 1809.3984150755555, "total_per_gpu": 3622.771787458213, "tpot_ms": 34.13145826195007, "tpot_relative": 2.726407, "ttft_ms": 400.1661081774728, "ttft_relative": 4.300303, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.34, "tpot_shape_error_pct": 24.24, "ttft_mape_pct": 34.48, "ttft_shape_error_pct": 51.71}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.12, "tpot_shape_error_pct": 16.89, "ttft_mape_pct": 22.56, "ttft_shape_error_pct": 16.77}, "framework": "vllm", "id": "bc6105dd8882252e", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 23315.53329933028, "interactivity_tok_s": 44.231933265373065, "output_per_gpu": 1404.042514478548, "total_per_gpu": null, "tpot_error_pct": 3.79, "tpot_ms": 22.608100667913813, "tpot_relative": 0.962104, "ttft_error_pct": 0.38, "ttft_ms": 187.44631605444843, "ttft_relative": 1.003755, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25375.4829497882, "interactivity_tok_s": 36.48271039182529, "output_per_gpu": 1127.5290167340663, "total_per_gpu": 2257.5350333741203, "tpot_error_pct": 16.65, "tpot_ms": 27.41024417484263, "tpot_relative": 1.166463, "ttft_error_pct": 12.81, "ttft_ms": 162.82981983671888, "ttft_relative": 0.871936, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 21796.02058529473, "interactivity_tok_s": 42.555734949657534, "output_per_gpu": 1313.0762281980542, "total_per_gpu": 2629.037073683584, "tpot_ms": 23.498595458002953, "tpot_relative": 1.0, "ttft_ms": 186.74515480626042, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26636.713815460807, "interactivity_tok_s": 38.70352247498252, "output_per_gpu": 2457.9608600967113, "total_per_gpu": null, "tpot_error_pct": 20.66, "tpot_ms": 25.837441557067258, "tpot_relative": 1.099531, "ttft_error_pct": 40.69, "ttft_ms": 205.01110258100425, "ttft_relative": 1.097812, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31433.83895937298, "interactivity_tok_s": 29.58525232989532, "output_per_gpu": 1828.828449775909, "total_per_gpu": 3655.889342787002, "tpot_error_pct": 3.8, "tpot_ms": 33.80062433976673, "tpot_relative": 1.43841, "ttft_error_pct": 27.07, "ttft_ms": 252.11954151406294, "ttft_relative": 1.350073, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 30393.342611938806, "interactivity_tok_s": 30.708661901550155, "output_per_gpu": 1896.8413820651508, "total_per_gpu": 3791.8494730870316, "tpot_ms": 32.56410205061786, "tpot_relative": 1.385789, "ttft_ms": 345.6824354235323, "ttft_relative": 1.851092, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35146.76160218842, "interactivity_tok_s": 29.296166625949663, "output_per_gpu": 3725.6348531367044, "total_per_gpu": null, "tpot_error_pct": 33.56, "tpot_ms": 34.134158668876154, "tpot_relative": 1.452604, "ttft_error_pct": 62.36, "ttft_ms": 227.5172839281159, "ttft_relative": 1.21833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43140.89181135483, "interactivity_tok_s": 21.60477385204848, "output_per_gpu": 2677.9590451830627, "total_per_gpu": 5353.451210597241, "tpot_error_pct": 9.91, "tpot_ms": 46.286066535483954, "tpot_relative": 1.969738, "ttft_error_pct": 27.8, "ttft_ms": 436.4199186478524, "ttft_relative": 2.336981, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 48091.50422929588, "interactivity_tok_s": 19.463413280614283, "output_per_gpu": 2412.0592577390144, "total_per_gpu": 4821.896577022708, "tpot_ms": 51.37844968826758, "tpot_relative": 2.186448, "ttft_ms": 604.4531204217947, "ttft_relative": 3.236781, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 56}, {"aic": {"coverage_pct": 80.2, "points": 81, "status_counts": {"failed": 20, "success": 81, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 47.0, "tpot_shape_error_pct": 22.38, "ttft_mape_pct": 43.24, "ttft_shape_error_pct": 45.58}, "aisimulate": {"coverage_pct": 100.0, "points": 101, "status_counts": {"failed": 0, "success": 101, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.73, "tpot_shape_error_pct": 18.02, "ttft_mape_pct": 29.73, "ttft_shape_error_pct": 35.83}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 95.35, "points": 41, "status_counts": {"failed": 2, "success": 41, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 60.39, "tpot_shape_error_pct": 30.38, "ttft_mape_pct": 46.87, "ttft_shape_error_pct": 46.62}, "aisimulate": {"coverage_pct": 100.0, "points": 43, "status_counts": {"failed": 0, "success": 43, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 58.38, "tpot_shape_error_pct": 27.01, "ttft_mape_pct": 30.0, "ttft_shape_error_pct": 44.36}, "gpu": "b200", "precisions": ["fp4"], "rows": 43, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 89.02, "tpot_shape_error_pct": 19.9, "ttft_mape_pct": 40.57, "ttft_shape_error_pct": 38.39}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.6, "tpot_shape_error_pct": 17.56, "ttft_mape_pct": 19.95, "ttft_shape_error_pct": 14.56}, "framework": "dynamo-vllm", "id": "022ab979bd087749", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8504.822, "interactivity_tok_s": 130.49719431032233, "output_per_gpu": 20.097666666666665, "total_per_gpu": null, "tpot_error_pct": 64.47, "tpot_ms": 7.663, "tpot_relative": 1.644724, "ttft_error_pct": 90.31, "ttft_ms": 665.573, "ttft_relative": 1.903051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6917.36512409999, "interactivity_tok_s": 140.735663800808, "output_per_gpu": 22.414604002875063, "total_per_gpu": 200.64479876850734, "tpot_error_pct": 52.51, "tpot_ms": 7.105519475257975, "tpot_relative": 1.525071, "ttft_error_pct": 10.14, "ttft_ms": 314.2869928999995, "ttft_relative": 0.898631, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 4679.536689026281, "interactivity_tok_s": 214.63187523849638, "output_per_gpu": 49.69388699674389, "total_per_gpu": 297.803647138178, "tpot_ms": 4.659140208735594, "tpot_relative": 1.0, "ttft_ms": 349.7399379964918, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11661.8, "interactivity_tok_s": 93.03190994511117, "output_per_gpu": 57.15283333333333, "total_per_gpu": null, "tpot_error_pct": 89.92, "tpot_ms": 10.749, "tpot_relative": 2.307078, "ttft_error_pct": 36.29, "ttft_ms": 665.573, "ttft_relative": 1.903051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9365.361053074996, "interactivity_tok_s": 102.2448804308264, "output_per_gpu": 63.90376127757691, "total_per_gpu": 574.4567025538028, "tpot_error_pct": 72.81, "tpot_ms": 9.78044079846666, "tpot_relative": 2.099194, "ttft_error_pct": 23.01, "ttft_ms": 375.9803542749992, "ttft_relative": 1.075028, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 5694.721731019672, "interactivity_tok_s": 176.68678261402428, "output_per_gpu": 160.23581168066684, "total_per_gpu": 962.467336331605, "tpot_ms": 5.659732919493584, "tpot_relative": 1.214759, "ttft_ms": 488.34493029280566, "ttft_relative": 1.396309, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14296.025, "interactivity_tok_s": 75.05253677574302, "output_per_gpu": 92.16683333333333, "total_per_gpu": null, "tpot_error_pct": 102.8, "tpot_ms": 13.324, "tpot_relative": 2.859755, "ttft_error_pct": 25.57, "ttft_ms": 665.573, "ttft_relative": 1.903051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11666.047863299946, "interactivity_tok_s": 82.3581024319745, "output_per_gpu": 104.08498310017275, "total_per_gpu": 925.8840472407569, "tpot_error_pct": 84.81, "tpot_ms": 12.142096168691747, "tpot_relative": 2.606081, "ttft_error_pct": 18.05, "ttft_ms": 434.3858893749945, "ttft_relative": 1.242025, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 6610.591723467223, "interactivity_tok_s": 152.2044454516907, "output_per_gpu": 273.18522196554693, "total_per_gpu": 1624.3789672258672, "tpot_ms": 6.570110334375204, "tpot_relative": 1.410155, "ttft_ms": 530.0537492672447, "ttft_relative": 1.515565, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18050.435, "interactivity_tok_s": 58.84429798752501, "output_per_gpu": 144.45233333333334, "total_per_gpu": null, "tpot_error_pct": 98.89, "tpot_ms": 16.994, "tpot_relative": 3.647454, "ttft_error_pct": 10.12, "ttft_ms": 665.573, "ttft_relative": 1.903051, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14608.88829406249, "interactivity_tok_s": 64.92603191821934, "output_per_gpu": 162.36002849575905, "total_per_gpu": 1465.258618933446, "tpot_error_pct": 80.26, "tpot_ms": 15.402142568324482, "tpot_relative": 3.305791, "ttft_error_pct": 28.61, "ttft_ms": 528.6346746999986, "ttft_relative": 1.511508, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 8553.619082123623, "interactivity_tok_s": 117.03503142203026, "output_per_gpu": 414.2033371197848, "total_per_gpu": 2494.1720106858684, "tpot_ms": 8.544450220156591, "tpot_relative": 1.833911, "ttft_ms": 740.5093569861492, "ttft_relative": 2.117314, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 135.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 78.66, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 111.38, "tpot_shape_error_pct": null, "ttft_mape_pct": 63.85, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "501e6630db9af90d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 22155.734, "interactivity_tok_s": 47.60317989241681, "output_per_gpu": 280.5514, "total_per_gpu": null, "tpot_error_pct": 135.33, "tpot_ms": 21.007, "tpot_relative": 2.353348, "ttft_error_pct": 78.66, "ttft_ms": 665.573, "ttft_relative": 0.213353, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18497.95659451723, "interactivity_tok_s": 52.99822347412941, "output_per_gpu": 308.83044818093146, "total_per_gpu": 2783.729092106491, "tpot_error_pct": 111.38, "tpot_ms": 18.86855699018177, "tpot_relative": 2.113785, "ttft_error_pct": 63.85, "ttft_ms": 1127.7716266093762, "ttft_relative": 0.361514, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 11333.065821366108, "interactivity_tok_s": 112.02682994494236, "output_per_gpu": 623.9797120655614, "total_per_gpu": 4514.0642775639435, "tpot_ms": 8.926433074036535, "tpot_relative": 1.0, "ttft_ms": 3119.5819449218106, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 135.02, "tpot_shape_error_pct": 15.82, "ttft_mape_pct": 88.5, "ttft_shape_error_pct": 79.66}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 112.28, "tpot_shape_error_pct": 14.95, "ttft_mape_pct": 77.35, "ttft_shape_error_pct": 65.86}, "framework": "dynamo-vllm", "id": "eb0093042edfe56b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 32765.267, "interactivity_tok_s": 31.8694626808592, "output_per_gpu": 536.7164285714285, "total_per_gpu": null, "tpot_error_pct": 117.79, "tpot_ms": 31.378, "tpot_relative": 2.177926, "ttft_error_pct": 80.89, "ttft_ms": 665.573, "ttft_relative": 0.191105, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27385.08209681805, "interactivity_tok_s": 35.14043084029863, "output_per_gpu": 595.6139744719133, "total_per_gpu": 5358.984537011548, "tpot_error_pct": 97.52, "tpot_ms": 28.457249273483917, "tpot_relative": 1.975199, "ttft_error_pct": 66.23, "ttft_ms": 1175.9993733472686, "ttft_relative": 0.337663, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 16759.452474110367, "interactivity_tok_s": 69.40933351868016, "output_per_gpu": 1699.289409955228, "total_per_gpu": 8746.476521580853, "tpot_ms": 14.407284284481275, "tpot_relative": 1.0, "ttft_ms": 3482.75604038854, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 39814.76, "interactivity_tok_s": 26.130810839060338, "output_per_gpu": 880.0372857142858, "total_per_gpu": null, "tpot_error_pct": 152.25, "tpot_ms": 38.269, "tpot_relative": 2.656226, "ttft_error_pct": 96.11, "ttft_ms": 665.573, "ttft_relative": 0.191105, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33699.5508085389, "interactivity_tok_s": 29.0316519668412, "output_per_gpu": 965.1227783021919, "total_per_gpu": 8686.175120003643, "tpot_error_pct": 127.05, "tpot_ms": 34.44516354571074, "tpot_relative": 2.390816, "ttft_error_pct": 88.47, "ttft_ms": 1973.5507419162102, "ttft_relative": 0.566664, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 31094.179814456766, "interactivity_tok_s": 65.914955299647, "output_per_gpu": 1820.4416352520443, "total_per_gpu": 9379.406899414063, "tpot_ms": 15.171063917953614, "tpot_relative": 1.053013, "ttft_ms": 17121.628977870976, "ttft_relative": 4.916115, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 14}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 187.99, "tpot_shape_error_pct": null, "ttft_mape_pct": 98.1, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 154.1, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.66, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "040f9d49624102d2", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 53227.313, "interactivity_tok_s": 19.46282600233554, "output_per_gpu": 1147.1001875, "total_per_gpu": null, "tpot_error_pct": 187.99, "tpot_ms": 51.38, "tpot_relative": 2.87994, "ttft_error_pct": 98.1, "ttft_ms": 665.573, "ttft_relative": 0.018956, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 44671.415458982054, "interactivity_tok_s": 22.05889505428467, "output_per_gpu": 1278.0949881550453, "total_per_gpu": 11500.495832269771, "tpot_error_pct": 154.1, "tpot_ms": 45.33318634224892, "tpot_relative": 2.541005, "ttft_error_pct": 91.66, "ttft_ms": 2928.4522043962834, "ttft_relative": 0.083402, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30828238642", "measured": {"e2e_ms": 51538.14144880082, "interactivity_tok_s": 56.0517661341995, "output_per_gpu": 2199.7096581624855, "total_per_gpu": 9917.767407667605, "tpot_ms": 17.840651043997322, "tpot_relative": 1.0, "ttft_ms": 35112.3024060535, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 126.6, "tpot_shape_error_pct": null, "ttft_mape_pct": 72.66, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 106.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 56.53, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "18c14feced932c02", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 28152.56, "interactivity_tok_s": 37.217611373702034, "output_per_gpu": 365.527, "total_per_gpu": null, "tpot_error_pct": 126.6, "tpot_ms": 26.869, "tpot_relative": 2.266025, "ttft_error_pct": 72.66, "ttft_ms": 665.573, "ttft_relative": 0.273434, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23529.891800317215, "interactivity_tok_s": 40.89414373149311, "output_per_gpu": 405.24781967580276, "total_per_gpu": 3658.610483608763, "tpot_error_pct": 106.23, "tpot_ms": 24.45337910889884, "tpot_relative": 2.062301, "ttft_error_pct": 56.53, "ttft_ms": 1058.0316287796845, "ttft_relative": 0.434666, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828238642", "measured": {"e2e_ms": 13331.783792218084, "interactivity_tok_s": 84.33603982212811, "output_per_gpu": 1063.201755209429, "total_per_gpu": 6406.37825476833, "tpot_ms": 11.857326975621396, "tpot_relative": 1.0, "ttft_ms": 2434.125249572753, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 12}, {"aic": {"coverage_pct": 66.67, "points": 2, "status_counts": {"failed": 1, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.21, "tpot_shape_error_pct": 9.66, "ttft_mape_pct": 60.6, "ttft_shape_error_pct": 37.09}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.5, "tpot_shape_error_pct": 10.28, "ttft_mape_pct": 17.36, "ttft_shape_error_pct": 23.03}, "framework": "vllm", "id": "390569ed742834e4", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 57483.948703094706, "interactivity_tok_s": 18.047405815031773, "output_per_gpu": 1138.9614227471338, "total_per_gpu": null, "tpot_error_pct": 20.37, "tpot_ms": 55.409625640882695, "tpot_relative": 0.79633, "ttft_error_pct": 51.63, "ttft_ms": 799.9016724717097, "ttft_relative": 0.483694, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 95383.77159208227, "interactivity_tok_s": 9.85032108368594, "output_per_gpu": 612.6252078546668, "total_per_gpu": 5512.041618545772, "tpot_error_pct": 45.9, "tpot_ms": 101.51953337401315, "tpot_relative": 1.459007, "ttft_error_pct": 3.9, "ttft_ms": 1589.260333281253, "ttft_relative": 0.961013, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 65936.08798572033, "interactivity_tok_s": 14.371685302301554, "output_per_gpu": 886.3476348918288, "total_per_gpu": 7974.835167380805, "tpot_ms": 69.58126197209836, "tpot_relative": 1.0, "ttft_ms": 1653.7340465805755, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 90271.4815576952, "interactivity_tok_s": 11.442999520373071, "output_per_gpu": 1450.557781266831, "total_per_gpu": null, "tpot_error_pct": 28.06, "tpot_ms": 87.38967420383125, "tpot_relative": 1.255937, "ttft_error_pct": 69.57, "ttft_ms": 871.8448471758246, "ttft_relative": 0.527198, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 145702.0072985538, "interactivity_tok_s": 6.46629850823645, "output_per_gpu": 803.5673699623724, "total_per_gpu": 7232.164708093152, "tpot_error_pct": 27.3, "tpot_ms": 154.64797963259036, "tpot_relative": 2.222552, "ttft_error_pct": 1.96, "ttft_ms": 2808.887296747068, "ttft_relative": 1.698512, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 115132.8155008609, "interactivity_tok_s": 8.231886369954895, "output_per_gpu": 1017.6618139269552, "total_per_gpu": 9159.030257539225, "tpot_ms": 121.4788391212304, "tpot_relative": 1.745856, "ttft_ms": 2864.9634357611376, "ttft_relative": 1.732421, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 282228.6187226373, "interactivity_tok_s": 3.840164381388797, "output_per_gpu": 829.1839653415467, "total_per_gpu": 7461.125210545438, "tpot_error_pct": 57.31, "tpot_ms": 260.40551931746984, "tpot_relative": 3.742466, "ttft_error_pct": 46.22, "ttft_ms": 41482.562151837905, "ttft_relative": 25.08418, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 230222.55290342064, "interactivity_tok_s": 6.0408595889408305, "output_per_gpu": 1012.5330863103461, "total_per_gpu": 9110.928880142654, "tpot_ms": 165.53935500019364, "tpot_relative": 2.37908, "ttft_ms": 77132.64058776395, "ttft_relative": 46.641502, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.56, "tpot_shape_error_pct": 10.91, "ttft_mape_pct": 64.21, "ttft_shape_error_pct": 45.76}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.21, "tpot_shape_error_pct": 9.73, "ttft_mape_pct": 9.1, "ttft_shape_error_pct": 10.27}, "framework": "vllm", "id": "5c7f56aaed7c9dfe", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 33472.09721899821, "interactivity_tok_s": 30.87974828397673, "output_per_gpu": 489.0043158308524, "total_per_gpu": null, "tpot_error_pct": 31.09, "tpot_ms": 32.38368366230797, "tpot_relative": 1.310932, "ttft_error_pct": 48.5, "ttft_ms": 343.5888324571565, "ttft_relative": 0.514978, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30810.897504712506, "interactivity_tok_s": 30.477011785926845, "output_per_gpu": 469.8292184443651, "total_per_gpu": 4234.936261657081, "tpot_error_pct": 32.83, "tpot_ms": 32.8116157523607, "tpot_relative": 1.328255, "ttft_error_pct": 14.92, "ttft_ms": 567.6300214812496, "ttft_relative": 0.850775, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 23448.3124649922, "interactivity_tok_s": 40.48125737300656, "output_per_gpu": 619.0541531861319, "total_per_gpu": 5580.016691890376, "tpot_ms": 24.702790004413583, "tpot_relative": 1.0, "ttft_ms": 667.1913910343562, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45568.85115543694, "interactivity_tok_s": 22.634815623811903, "output_per_gpu": 718.3854578281196, "total_per_gpu": null, "tpot_error_pct": 22.25, "tpot_ms": 44.179728106466065, "tpot_relative": 1.788451, "ttft_error_pct": 65.23, "ttft_ms": 372.98930252215285, "ttft_relative": 0.559044, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42708.254672755786, "interactivity_tok_s": 22.062911841159412, "output_per_gpu": 677.5457408645158, "total_per_gpu": 6116.938402369388, "tpot_error_pct": 25.42, "tpot_ms": 45.324932955334226, "tpot_relative": 1.83481, "ttft_error_pct": 7.92, "ttft_ms": 987.7403156921835, "ttft_relative": 1.480445, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 34353.12933823516, "interactivity_tok_s": 27.671135083591754, "output_per_gpu": 845.1932874112678, "total_per_gpu": 7630.474173734967, "tpot_ms": 36.13874157959546, "tpot_relative": 1.462942, "ttft_ms": 1072.7500648492169, "ttft_relative": 1.60786, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 66438.11599299173, "interactivity_tok_s": 15.492941645609745, "output_per_gpu": 985.4584077445296, "total_per_gpu": null, "tpot_error_pct": 11.34, "tpot_ms": 64.54552162360795, "tpot_relative": 2.612884, "ttft_error_pct": 78.9, "ttft_ms": 408.04737204080027, "ttft_relative": 0.61159, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 63042.610313946694, "interactivity_tok_s": 15.082203205429058, "output_per_gpu": 922.9706270375417, "total_per_gpu": 8304.347329653328, "tpot_error_pct": 14.38, "tpot_ms": 66.30331035720533, "tpot_relative": 2.684041, "ttft_error_pct": 4.45, "ttft_ms": 1848.0145094921838, "ttft_relative": 2.769842, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 55446.72353100195, "interactivity_tok_s": 17.250512541292746, "output_per_gpu": 1052.8937648552346, "total_per_gpu": 9473.319375989808, "tpot_ms": 57.969292077918766, "tpot_relative": 2.34667, "ttft_ms": 1934.0376610703117, "ttft_relative": 2.898775, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 83.33, "points": 5, "status_counts": {"failed": 1, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.79, "tpot_shape_error_pct": 15.68, "ttft_mape_pct": 39.84, "ttft_shape_error_pct": 43.25}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.26, "tpot_shape_error_pct": 11.51, "ttft_mape_pct": 24.16, "ttft_shape_error_pct": 26.6}, "framework": "vllm", "id": "396dfa28c0e0acfc", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7463.605366370731, "interactivity_tok_s": 140.94334916179824, "output_per_gpu": 34.266281166519065, "total_per_gpu": null, "tpot_error_pct": 56.77, "tpot_ms": 7.095049223302006, "tpot_relative": 1.567722, "ttft_error_pct": 9.6, "ttft_ms": 205.37001093277956, "ttft_relative": 1.096042, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6748.332940999993, "interactivity_tok_s": 140.73566380080794, "output_per_gpu": 34.46406720494975, "total_per_gpu": 308.50582183805176, "tpot_error_pct": 57.0, "tpot_ms": 7.105519475257978, "tpot_relative": 1.570036, "ttft_error_pct": 22.48, "ttft_ms": 145.2548098000002, "ttft_relative": 0.775212, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 4393.1547585409135, "interactivity_tok_s": 220.9599989646796, "output_per_gpu": 52.93455268383834, "total_per_gpu": 473.8447607545224, "tpot_ms": 4.525706031343029, "tpot_relative": 1.0, "ttft_ms": 187.37418483942747, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8725.822981741316, "interactivity_tok_s": 120.46298584197176, "output_per_gpu": 58.61911261210637, "total_per_gpu": null, "tpot_error_pct": 63.58, "tpot_ms": 8.301305110532796, "tpot_relative": 1.834256, "ttft_error_pct": 13.43, "ttft_ms": 233.58785366626614, "ttft_relative": 1.246638, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7759.978801850006, "interactivity_tok_s": 122.99445580812821, "output_per_gpu": 60.09350864223658, "total_per_gpu": 535.5213505514981, "tpot_error_pct": 60.22, "tpot_ms": 8.130447778556812, "tpot_relative": 1.796504, "ttft_error_pct": 20.93, "ttft_ms": 162.83662315000012, "ttft_relative": 0.869045, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 4947.87116383668, "interactivity_tok_s": 197.0581578281664, "output_per_gpu": 94.23245553141854, "total_per_gpu": 839.7494669915329, "tpot_ms": 5.074644008760066, "tpot_relative": 1.121293, "ttft_ms": 205.93767155660316, "ttft_relative": 1.099072, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11040.434397957313, "interactivity_tok_s": 94.89227552465549, "output_per_gpu": 92.65939754954516, "total_per_gpu": null, "tpot_error_pct": 74.17, "tpot_ms": 10.538265569784697, "tpot_relative": 2.328535, "ttft_error_pct": 11.26, "ttft_ms": 259.7887200675678, "ttft_relative": 1.38647, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9669.037420500012, "interactivity_tok_s": 96.82700453379782, "output_per_gpu": 93.14848867703599, "total_per_gpu": 837.3493607184008, "tpot_error_pct": 70.69, "tpot_ms": 10.327697369290675, "tpot_relative": 2.282008, "ttft_error_pct": 24.13, "ttft_ms": 177.16223169999984, "ttft_relative": 0.9455, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 5795.143396657659, "interactivity_tok_s": 165.26982101980963, "output_per_gpu": 155.31135692818305, "total_per_gpu": 1396.1564732094553, "tpot_ms": 6.050711459777872, "tpot_relative": 1.336965, "ttft_ms": 233.49852046230808, "ttft_relative": 1.246162, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 71652.81015671989, "interactivity_tok_s": 14.365440172146187, "output_per_gpu": 913.73945916146, "total_per_gpu": null, "tpot_error_pct": 26.73, "tpot_ms": 69.61151123924111, "tpot_relative": 15.381359, "ttft_error_pct": 77.17, "ttft_ms": 440.2341589762415, "ttft_relative": 2.349492, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 67438.23742792613, "interactivity_tok_s": 14.112098001948546, "output_per_gpu": 863.6311499558706, "total_per_gpu": 7770.445584992327, "tpot_error_pct": 29.01, "tpot_ms": 70.86118590318206, "tpot_relative": 15.657488, "ttft_error_pct": 5.44, "ttft_ms": 2033.4225111203127, "ttft_relative": 10.852202, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 52634.88873463539, "interactivity_tok_s": 18.2053632316654, "output_per_gpu": 1109.870741113632, "total_per_gpu": 9985.96472653778, "tpot_ms": 54.928868338131004, "tpot_relative": 12.137083, "ttft_ms": 1928.5246253170044, "ttft_relative": 10.292371, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 119151.17931153064, "interactivity_tok_s": 8.619361073765807, "output_per_gpu": 1098.9735960366456, "total_per_gpu": null, "tpot_error_pct": 12.7, "tpot_ms": 116.01788014701407, "tpot_relative": 25.635311, "ttft_error_pct": 87.75, "ttft_ms": 464.8879211352469, "ttft_relative": 2.481067, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 111957.52501685098, "interactivity_tok_s": 8.550783949274685, "output_per_gpu": 1043.1097946920338, "total_per_gpu": 9388.063933197374, "tpot_error_pct": 13.6, "tpot_ms": 116.94834133715008, "tpot_relative": 25.840905, "ttft_error_pct": 6.12, "ttft_ms": 4026.8452118886794, "ttft_relative": 21.490929, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 98776.26515064525, "interactivity_tok_s": 9.713944515701197, "output_per_gpu": 1185.5232266398657, "total_per_gpu": 10669.795166932965, "tpot_ms": 102.94479224002603, "tpot_relative": 22.746681, "ttft_ms": 3794.7847061847824, "ttft_relative": 20.252441, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 209686.6696724414, "interactivity_tok_s": 5.047824302693088, "output_per_gpu": 1117.380303009954, "total_per_gpu": 10054.360307269611, "tpot_error_pct": 59.04, "tpot_ms": 198.10515185056767, "tpot_relative": 43.773314, "ttft_error_pct": 65.88, "ttft_ms": 26969.895397834414, "ttft_relative": 143.936025, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 194035.71949128507, "interactivity_tok_s": 8.028188030949382, "output_per_gpu": 1197.0087987369225, "total_per_gpu": 10770.869793438429, "tpot_ms": 124.5611084524815, "tpot_relative": 27.523022, "ttft_ms": 79035.15879779725, "ttft_relative": 421.803883, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 73.86, "tpot_shape_error_pct": 15.38, "ttft_mape_pct": 29.77, "ttft_shape_error_pct": 28.18}, "aisimulate": {"points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.97, "tpot_shape_error_pct": 13.93, "ttft_mape_pct": 4.85, "ttft_shape_error_pct": 4.01}, "framework": "vllm", "id": "b7eec04cb49ca9b3", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 8630.516945734114, "interactivity_tok_s": 122.27930687422571, "output_per_gpu": 59.2664382928793, "total_per_gpu": null, "tpot_error_pct": 56.14, "tpot_ms": 8.177998596512998, "tpot_relative": 1.561428, "ttft_error_pct": 21.92, "ttft_ms": 264.42438150131744, "ttft_relative": 1.219211, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7805.851871700191, "interactivity_tok_s": 122.18685025885766, "output_per_gpu": 59.5899086538374, "total_per_gpu": 533.4203195804539, "tpot_error_pct": 56.26, "tpot_ms": 8.184186742529663, "tpot_relative": 1.56261, "ttft_error_pct": 7.63, "ttft_ms": 200.32958910000033, "ttft_relative": 0.923682, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 5084.162555192597, "interactivity_tok_s": 190.93034381650827, "output_per_gpu": 91.48089877986251, "total_per_gpu": 818.893187874001, "tpot_ms": 5.23751217334548, "tpot_relative": 1.0, "ttft_ms": 216.8814758071676, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10567.974586250468, "interactivity_tok_s": 99.63452799245644, "output_per_gpu": 96.8018981925809, "total_per_gpu": null, "tpot_error_pct": 65.08, "tpot_ms": 10.036681260493474, "tpot_relative": 1.916307, "ttft_error_pct": 29.21, "ttft_ms": 300.44965676564465, "ttft_relative": 1.385317, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9382.203197750025, "interactivity_tok_s": 102.01559137839068, "output_per_gpu": 99.41910018033172, "total_per_gpu": 885.9700823287525, "tpot_error_pct": 61.23, "tpot_ms": 9.802423203046036, "tpot_relative": 1.87158, "ttft_error_pct": 4.32, "ttft_ms": 222.4905110000014, "ttft_relative": 1.025862, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 5914.62583396351, "interactivity_tok_s": 164.47605975911344, "output_per_gpu": 157.7141362750873, "total_per_gpu": 1405.4644031840699, "tpot_ms": 6.079912185789039, "tpot_relative": 1.16084, "ttft_ms": 232.53632097039372, "ttft_relative": 1.072182, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14248.420983934442, "interactivity_tok_s": 73.52025945467558, "output_per_gpu": 143.5948588483546, "total_per_gpu": null, "tpot_error_pct": 84.12, "tpot_ms": 13.60169302199605, "tpot_relative": 2.596976, "ttft_error_pct": 32.23, "ttft_ms": 333.88902243248145, "ttft_relative": 1.5395, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12461.843804500015, "interactivity_tok_s": 75.21247763265247, "output_per_gpu": 144.75695247236243, "total_per_gpu": 1301.278671654489, "tpot_error_pct": 79.98, "tpot_ms": 13.295666244158719, "tpot_relative": 2.538546, "ttft_error_pct": 4.01, "ttft_ms": 242.37809475000034, "ttft_relative": 1.11756, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 7041.396071697818, "interactivity_tok_s": 135.36764601033244, "output_per_gpu": 255.5409134379915, "total_per_gpu": 2297.160410692207, "tpot_ms": 7.387289573785388, "tpot_relative": 1.410458, "ttft_ms": 252.51276416820474, "ttft_relative": 1.164289, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19995.439601729893, "interactivity_tok_s": 52.120161416851126, "output_per_gpu": 204.64666351451373, "total_per_gpu": null, "tpot_error_pct": 99.55, "tpot_ms": 19.186433288303036, "tpot_relative": 3.663272, "ttft_error_pct": 16.07, "ttft_ms": 367.71834779588545, "ttft_relative": 1.695481, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17736.08239662504, "interactivity_tok_s": 53.05753756446397, "output_per_gpu": 206.06269172501163, "total_per_gpu": 1833.0229137478866, "tpot_error_pct": 96.03, "tpot_ms": 18.84746345012747, "tpot_relative": 3.598553, "ttft_error_pct": 9.49, "ttft_ms": 286.73614827500126, "ttft_relative": 1.322087, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 9221.978141122963, "interactivity_tok_s": 104.00686149240838, "output_per_gpu": 395.5831846350025, "total_per_gpu": 3518.8953209296897, "tpot_ms": 9.614750273692199, "tpot_relative": 1.835748, "ttft_ms": 316.80655121745076, "ttft_relative": 1.460736, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28252.23384516225, "interactivity_tok_s": 36.7324959197297, "output_per_gpu": 289.67620913987946, "total_per_gpu": null, "tpot_error_pct": 100.53, "tpot_ms": 27.22385111497097, "tpot_relative": 5.197859, "ttft_error_pct": 4.7, "ttft_ms": 402.23415454694515, "ttft_relative": 1.854627, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25026.400526599802, "interactivity_tok_s": 37.10417319737309, "output_per_gpu": 286.22221318605324, "total_per_gpu": 2583.0838334204, "tpot_error_pct": 98.53, "tpot_ms": 26.95114629506953, "tpot_relative": 5.145792, "ttft_error_pct": 2.68, "ttft_ms": 373.87330760625207, "ttft_relative": 1.72386, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 12807.946552711655, "interactivity_tok_s": 73.66138738854802, "output_per_gpu": 558.0364374363202, "total_per_gpu": 5036.139172972866, "tpot_ms": 13.575633523235375, "tpot_relative": 2.592, "ttft_ms": 384.1881538712187, "ttft_relative": 1.77142, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35659.00410861744, "interactivity_tok_s": 29.044687878181964, "output_per_gpu": 459.0145016429236, "total_per_gpu": null, "tpot_error_pct": 77.97, "tpot_ms": 34.42970377902351, "tpot_relative": 6.573675, "ttft_error_pct": 22.82, "ttft_ms": 437.41714267638815, "ttft_relative": 2.016849, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32373.84724292807, "interactivity_tok_s": 29.049229288541063, "output_per_gpu": 447.7062482788725, "total_per_gpu": 4003.5318249067227, "tpot_error_pct": 77.94, "tpot_ms": 34.42432121235196, "tpot_relative": 6.572647, "ttft_error_pct": 7.57, "ttft_ms": 523.9039265906252, "ttft_relative": 2.415623, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 18476.144519903028, "interactivity_tok_s": 51.68945358148032, "output_per_gpu": 784.8716403336539, "total_per_gpu": 7018.572116476783, "tpot_ms": 19.34630627161993, "tpot_relative": 3.693797, "ttft_ms": 566.7823874049645, "ttft_relative": 2.613328, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 50455.87976514086, "interactivity_tok_s": 20.467677395609037, "output_per_gpu": 648.8044634714063, "total_per_gpu": null, "tpot_error_pct": 64.53, "tpot_ms": 48.85752206620823, "tpot_relative": 9.328383, "ttft_error_pct": 45.44, "ttft_ms": 474.63469140983926, "ttft_relative": 2.188452, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46532.51023864072, "interactivity_tok_s": 20.178436328665256, "output_per_gpu": 622.9040909905187, "total_per_gpu": 5614.7191764802155, "tpot_error_pct": 66.88, "tpot_ms": 49.557853924459515, "tpot_relative": 9.462098, "ttft_error_pct": 1.77, "ttft_ms": 854.5089403937494, "ttft_relative": 3.939981, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 28256.20431536845, "interactivity_tok_s": 33.67477806353377, "output_per_gpu": 1028.3483453170202, "total_per_gpu": 9269.303666591359, "tpot_ms": 29.695815607553904, "tpot_relative": 5.669832, "ttft_ms": 869.9464718243689, "ttft_relative": 4.011161, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 69534.77410642992, "interactivity_tok_s": 14.822168329931623, "output_per_gpu": 941.5720528521249, "total_per_gpu": null, "tpot_error_pct": 42.92, "tpot_ms": 67.46651216884494, "tpot_relative": 12.881404, "ttft_error_pct": 65.78, "ttft_ms": 516.5321577015366, "ttft_relative": 2.381633, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 65333.58785340722, "interactivity_tok_s": 14.416944451643722, "output_per_gpu": 886.6807923226027, "total_per_gpu": 8005.02676362628, "tpot_error_pct": 46.94, "tpot_ms": 69.36282534444993, "tpot_relative": 13.243468, "ttft_error_pct": 1.33, "ttft_ms": 1489.3370577390624, "ttft_relative": 6.867055, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 44978.2757682613, "interactivity_tok_s": 21.18420662984629, "output_per_gpu": 1292.7560855928848, "total_per_gpu": 11671.107746570715, "tpot_ms": 47.20497762663939, "tpot_relative": 9.012863, "ttft_ms": 1509.4810311840774, "ttft_relative": 6.959935, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 8, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.55, "tpot_shape_error_pct": 17.84, "ttft_mape_pct": 13.36, "ttft_shape_error_pct": 26.07}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 81.07, "tpot_shape_error_pct": 1.99, "ttft_mape_pct": 5.36, "ttft_shape_error_pct": 2.29}, "framework": "vllm", "id": "5a65c17bbad92286", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 30632.6485590683, "interactivity_tok_s": 34.39182393299749, "output_per_gpu": 534.3318573461881, "total_per_gpu": null, "tpot_error_pct": 27.04, "tpot_ms": 29.076678281099902, "tpot_relative": 0.729563, "ttft_error_pct": 2.5, "ttft_ms": 887.2066775031005, "ttft_relative": 1.025001, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 68111.4590434582, "interactivity_tok_s": 13.719648389505384, "output_per_gpu": 213.9662318707164, "total_per_gpu": 1931.7046534308797, "tpot_error_pct": 82.88, "tpot_ms": 72.88816532389659, "tpot_relative": 1.828837, "ttft_error_pct": 6.57, "ttft_ms": 922.4638168640656, "ttft_relative": 1.065734, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 37587.76347833205, "interactivity_tok_s": 25.09100291928, "output_per_gpu": 385.3058780332095, "total_per_gpu": 3478.5730023078795, "tpot_ms": 39.854923424826396, "tpot_relative": 1.0, "ttft_ms": 865.5669888953952, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 40196.27613702215, "interactivity_tok_s": 26.075588261750887, "output_per_gpu": 814.4037992078829, "total_per_gpu": null, "tpot_error_pct": 40.06, "tpot_ms": 38.3500456427614, "tpot_relative": 0.962241, "ttft_error_pct": 24.23, "ttft_ms": 964.1794444772408, "ttft_relative": 1.113928, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 107348.2733317706, "interactivity_tok_s": 8.719963968956176, "output_per_gpu": 272.9717931112286, "total_per_gpu": 2456.0397858703554, "tpot_error_pct": 79.25, "tpot_ms": 114.67937293778807, "tpot_relative": 2.87742, "ttft_error_pct": 4.14, "ttft_ms": 1325.0826444171937, "ttft_relative": 1.530884, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 60390.35639083515, "interactivity_tok_s": 15.630378097783055, "output_per_gpu": 483.4906286196812, "total_per_gpu": 4350.164558949659, "tpot_ms": 63.97797889110793, "tpot_relative": 1.605272, "ttft_ms": 1272.4451302042742, "ttft_relative": 1.470071, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 71.72, "tpot_shape_error_pct": 240.88, "ttft_mape_pct": 54.13, "ttft_shape_error_pct": 173.88}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 70.81, "tpot_shape_error_pct": 236.55, "ttft_mape_pct": 69.11, "ttft_shape_error_pct": 172.5}, "framework": "vllm", "id": "ea748d4648783a2c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7098.400944836382, "interactivity_tok_s": 148.06118435427484, "output_per_gpu": 18.014620615791028, "total_per_gpu": null, "tpot_error_pct": 69.83, "tpot_ms": 6.753964615109658, "tpot_relative": 0.301663, "ttft_error_pct": 78.07, "ttft_ms": 189.09514357920258, "ttft_relative": 0.219302, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6399.628049399427, "interactivity_tok_s": 148.20857105495497, "output_per_gpu": 18.170977922836155, "total_per_gpu": 162.6578907343979, "tpot_error_pct": 69.86, "tpot_ms": 6.747248103682243, "tpot_relative": 0.301363, "ttft_error_pct": 84.99, "ttft_ms": 129.4106907999999, "ttft_relative": 0.150083, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 21597.923991363496, "interactivity_tok_s": 44.664627198546185, "output_per_gpu": 5.384059565656076, "total_per_gpu": 48.195522346509236, "tpot_ms": 22.389081981021203, "tpot_relative": 1.0, "ttft_ms": 862.2587647289038, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7899.512220920254, "interactivity_tok_s": 133.12775309789103, "output_per_gpu": 32.37541671531289, "total_per_gpu": null, "tpot_error_pct": 65.58, "tpot_ms": 7.511581745578502, "tpot_relative": 0.335502, "ttft_error_pct": 4.03, "ttft_ms": 215.1640951934472, "ttft_relative": 0.249535, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7029.688410849994, "interactivity_tok_s": 135.72696279403553, "output_per_gpu": 33.16344910043991, "total_per_gpu": 295.53499957782566, "tpot_error_pct": 62.41, "tpot_ms": 7.36773283225597, "tpot_relative": 0.329077, "ttft_error_pct": 35.27, "ttft_ms": 145.12037989999936, "ttft_relative": 0.168303, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 4463.867339293938, "interactivity_tok_s": 220.43013609266237, "output_per_gpu": 52.21927339290129, "total_per_gpu": 465.3503588660424, "tpot_ms": 4.536584777952636, "tpot_relative": 0.202625, "ttft_ms": 224.20845533488318, "ttft_relative": 0.260025, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9880.84970036328, "interactivity_tok_s": 106.1035127409383, "output_per_gpu": 51.76680300897545, "total_per_gpu": null, "tpot_error_pct": 79.75, "tpot_ms": 9.424758654707258, "tpot_relative": 0.420953, "ttft_error_pct": 80.3, "ttft_ms": 239.32159659775658, "ttft_relative": 0.277552, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8643.20637887504, "interactivity_tok_s": 108.30624131207814, "output_per_gpu": 52.07114833446366, "total_per_gpu": 468.0885690041835, "tpot_error_pct": 80.17, "tpot_ms": 9.233078240787233, "tpot_relative": 0.412392, "ttft_error_pct": 87.05, "ttft_ms": 157.29093367499928, "ttft_relative": 0.182417, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588412", "measured": {"e2e_ms": 44001.74831414479, "interactivity_tok_s": 21.481272604262212, "output_per_gpu": 10.200923784746394, "total_per_gpu": 91.70022113305986, "tpot_ms": 46.55217679243011, "tpot_relative": 2.079236, "ttft_ms": 1214.7699042456225, "ttft_relative": 1.408823, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.96, "tpot_shape_error_pct": 12.99, "ttft_mape_pct": 43.62, "ttft_shape_error_pct": 35.55}, "aisimulate": {"points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.18, "tpot_shape_error_pct": 12.43, "ttft_mape_pct": 40.26, "ttft_shape_error_pct": 86.32}, "framework": "vllm", "id": "429c52fb80bb0034", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7231.6667482440525, "interactivity_tok_s": 144.85633281670457, "output_per_gpu": 17.682645571444482, "total_per_gpu": null, "tpot_error_pct": 18.85, "tpot_ms": 6.903391660931802, "tpot_relative": 1.188455, "ttft_error_pct": 48.61, "ttft_ms": 169.4970791108188, "ttft_relative": 0.513862, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6523.510349400003, "interactivity_tok_s": 144.93267049713967, "output_per_gpu": 17.82590871656937, "total_per_gpu": 159.5689964829658, "tpot_error_pct": 18.78, "tpot_ms": 6.899755566290594, "tpot_relative": 1.187829, "ttft_error_pct": 66.2, "ttft_ms": 111.49384569999992, "ttft_relative": 0.338015, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 5727.739845700307, "interactivity_tok_s": 172.15518912677373, "output_per_gpu": 20.30069207738503, "total_per_gpu": 181.72207174420248, "tpot_ms": 5.808712505689316, "tpot_relative": 1.0, "ttft_ms": 329.8492918000193, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7464.589575001583, "interactivity_tok_s": 140.68873699889437, "output_per_gpu": 34.261763145892154, "total_per_gpu": null, "tpot_error_pct": 9.36, "tpot_ms": 7.1078895250005605, "tpot_relative": 1.22366, "ttft_error_pct": 34.93, "ttft_ms": 193.2185909260098, "ttft_relative": 0.585778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6650.75455854999, "interactivity_tok_s": 143.20643860670188, "output_per_gpu": 35.044943634205765, "total_per_gpu": 312.30187700840014, "tpot_error_pct": 7.44, "tpot_ms": 6.982926254778053, "tpot_relative": 1.202147, "ttft_error_pct": 57.64, "ttft_ms": 125.78055704999913, "ttft_relative": 0.381327, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 6367.66606164997, "interactivity_tok_s": 153.85787776863347, "output_per_gpu": 36.590286963346806, "total_per_gpu": 326.0731538964617, "tpot_ms": 6.499504702019664, "tpot_relative": 1.118923, "ttft_ms": 296.92192049997175, "ttft_relative": 0.900174, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10182.260033237055, "interactivity_tok_s": 102.63286811027551, "output_per_gpu": 50.234427163552645, "total_per_gpu": null, "tpot_error_pct": 34.75, "tpot_ms": 9.743467355170608, "tpot_relative": 1.677388, "ttft_error_pct": 18.96, "ttft_ms": 214.69292889752177, "ttft_relative": 0.650882, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8928.674931675185, "interactivity_tok_s": 104.53960797789976, "output_per_gpu": 50.45127656767503, "total_per_gpu": 453.5268878901813, "tpot_error_pct": 32.29, "tpot_ms": 9.565752343469715, "tpot_relative": 1.646794, "ttft_error_pct": 48.26, "ttft_ms": 137.05808665000114, "ttft_relative": 0.415517, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 6908.913545549876, "interactivity_tok_s": 138.29881548332673, "output_per_gpu": 65.08541540961139, "total_per_gpu": 585.0790684783854, "tpot_ms": 7.2307199197997445, "tpot_relative": 1.244806, "ttft_ms": 264.9100496749952, "ttft_relative": 0.803125, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11924.095200049134, "interactivity_tok_s": 87.52714742910926, "output_per_gpu": 85.79267297327387, "total_per_gpu": null, "tpot_error_pct": 27.53, "tpot_ms": 11.425026741673816, "tpot_relative": 1.966878, "ttft_error_pct": 25.91, "ttft_ms": 236.29284331682052, "ttft_relative": 0.716366, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10590.306149812503, "interactivity_tok_s": 88.75894013888636, "output_per_gpu": 86.05411132608123, "total_per_gpu": 765.4911064319182, "tpot_error_pct": 25.76, "tpot_ms": 11.266470717600289, "tpot_relative": 1.939581, "ttft_error_pct": 49.26, "ttft_ms": 161.84041472499993, "ttft_relative": 0.49065, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 8618.904427287485, "interactivity_tok_s": 111.62075803314971, "output_per_gpu": 105.72608281683269, "total_per_gpu": 940.4823879651283, "tpot_ms": 8.958907085212724, "tpot_relative": 1.542322, "ttft_ms": 318.92990851255263, "ttft_relative": 0.966896, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16168.197118114527, "interactivity_tok_s": 64.2993961871766, "output_per_gpu": 126.54472140914847, "total_per_gpu": null, "tpot_error_pct": 38.83, "tpot_ms": 15.552245577687597, "tpot_relative": 2.6774, "ttft_error_pct": 23.9, "ttft_ms": 258.24989214011606, "ttft_relative": 0.782933, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14257.014942856307, "interactivity_tok_s": 65.12172710378998, "output_per_gpu": 125.01277889870371, "total_per_gpu": 1128.2090392274815, "tpot_error_pct": 37.07, "tpot_ms": 15.355858090283997, "tpot_relative": 2.643591, "ttft_error_pct": 37.91, "ttft_ms": 210.69169903124967, "ttft_relative": 0.638751, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 10593.368689543671, "interactivity_tok_s": 89.26434826074426, "output_per_gpu": 168.0080476848859, "total_per_gpu": 1516.230578432614, "tpot_ms": 11.202680795685254, "tpot_relative": 1.9286, "ttft_ms": 339.3553994999138, "ttft_relative": 1.02882, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21232.28044055292, "interactivity_tok_s": 48.828719030384846, "output_per_gpu": 192.72541220699128, "total_per_gpu": null, "tpot_error_pct": 27.83, "tpot_ms": 20.479750848629184, "tpot_relative": 3.525695, "ttft_error_pct": 43.85, "ttft_ms": 281.49532240526435, "ttft_relative": 0.853406, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19132.501210315717, "interactivity_tok_s": 49.10782435742877, "output_per_gpu": 189.01429316196814, "total_per_gpu": 1690.2259929256807, "tpot_error_pct": 27.11, "tpot_ms": 20.363353764596688, "tpot_relative": 3.505657, "ttft_error_pct": 41.53, "ttft_ms": 293.1215945781255, "ttft_relative": 0.888653, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 15336.424051896875, "interactivity_tok_s": 62.41894925284965, "output_per_gpu": 235.88340370854667, "total_per_gpu": 2109.344502885403, "tpot_ms": 16.02077593374974, "tpot_relative": 2.75806, "ttft_ms": 501.30193324373516, "ttft_relative": 1.519791, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26213.40696643104, "interactivity_tok_s": 39.48621831895892, "output_per_gpu": 312.20665098895586, "total_per_gpu": null, "tpot_error_pct": 8.6, "tpot_ms": 25.325291774519208, "tpot_relative": 4.35988, "ttft_error_pct": 50.32, "ttft_ms": 305.6334810978911, "ttft_relative": 0.926585, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23890.72343834857, "interactivity_tok_s": 39.344863710543585, "output_per_gpu": 302.41345680266136, "total_per_gpu": 2725.887756549377, "tpot_error_pct": 8.99, "tpot_ms": 25.41627815404076, "tpot_relative": 4.375544, "ttft_error_pct": 24.99, "ttft_ms": 461.4277993156264, "ttft_relative": 1.398905, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 22121.02175714524, "interactivity_tok_s": 42.882153729446514, "output_per_gpu": 328.263892392896, "total_per_gpu": 2958.8978435405484, "tpot_ms": 23.31972424494424, "tpot_relative": 4.014612, "ttft_ms": 615.1657728797204, "ttft_relative": 1.86499, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37013.79366408121, "interactivity_tok_s": 27.88864841957218, "output_per_gpu": 442.21352041208826, "total_per_gpu": null, "tpot_error_pct": 0.67, "tpot_ms": 35.8568828777734, "tpot_relative": 6.172948, "ttft_error_pct": 66.31, "ttft_ms": 332.2024801190208, "ttft_relative": 1.007134, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34051.35416936015, "interactivity_tok_s": 27.683294913282403, "output_per_gpu": 424.721603093148, "total_per_gpu": 3834.4213941356174, "tpot_error_pct": 1.42, "tpot_ms": 36.12286771254969, "tpot_relative": 6.218739, "ttft_error_pct": 18.96, "ttft_ms": 799.07539891797, "ttft_relative": 2.422547, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 33789.640476939894, "interactivity_tok_s": 28.075826354070188, "output_per_gpu": 430.0616867215849, "total_per_gpu": 3882.632106193184, "tpot_ms": 35.61782963709735, "tpot_relative": 6.131794, "ttft_ms": 986.0280918023335, "ttft_relative": 2.989329, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 54924.44029832192, "interactivity_tok_s": 18.75001077655486, "output_per_gpu": 596.0188182563995, "total_per_gpu": null, "tpot_error_pct": 13.19, "tpot_ms": 53.33330268003935, "tpot_relative": 9.181605, "ttft_error_pct": 79.83, "ttft_ms": 364.471656641662, "ttft_relative": 1.104964, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 50399.50765614094, "interactivity_tok_s": 18.871414181747856, "output_per_gpu": 577.3881025324315, "total_per_gpu": 5194.998851511415, "tpot_error_pct": 13.74, "tpot_ms": 52.99019937611166, "tpot_relative": 9.122538, "ttft_error_pct": 17.62, "ttft_ms": 1488.678940140624, "ttft_relative": 4.513209, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28197744776", "measured": {"e2e_ms": 58525.00286276023, "interactivity_tok_s": 16.277551999494513, "output_per_gpu": 499.6797389715356, "total_per_gpu": 4495.824660562424, "tpot_ms": 61.43429921348457, "tpot_relative": 10.576233, "ttft_ms": 1807.0956772706866, "ttft_relative": 5.47855, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 9, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 68.97, "points": 40, "status_counts": {"failed": 18, "success": 40, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.29, "tpot_shape_error_pct": 13.46, "ttft_mape_pct": 39.51, "ttft_shape_error_pct": 44.41}, "aisimulate": {"coverage_pct": 100.0, "points": 58, "status_counts": {"failed": 0, "success": 58, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.09, "tpot_shape_error_pct": 10.28, "ttft_mape_pct": 29.52, "ttft_shape_error_pct": 28.49}, "gpu": "b300", "precisions": ["fp4"], "rows": 58, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 6.44, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.77, "tpot_shape_error_pct": null, "ttft_mape_pct": 25.62, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "cf0573e185d48618", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13145.117, "interactivity_tok_s": 81.45975887911372, "output_per_gpu": 120.021, "total_per_gpu": null, "tpot_error_pct": 43.05, "tpot_ms": 12.276, "tpot_relative": 1.430524, "ttft_error_pct": 6.44, "ttft_ms": 586.769, "ttft_relative": 0.935625, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10648.204413956262, "interactivity_tok_s": 89.79490173736987, "output_per_gpu": 133.0782845678697, "total_per_gpu": 1200.9982091192, "tpot_error_pct": 29.77, "tpot_ms": 11.13648971881252, "tpot_relative": 1.297737, "ttft_error_pct": 25.62, "ttft_ms": 466.47378278125143, "ttft_relative": 0.74381, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 8475.06766893639, "interactivity_tok_s": 116.53013154745234, "output_per_gpu": 208.16892814631402, "total_per_gpu": 1504.215154867849, "tpot_ms": 8.581471476266112, "tpot_relative": 1.0, "ttft_ms": 627.1411630121293, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.21, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.87, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "45ad88ae5ba12545", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 124633.50898789268, "interactivity_tok_s": 18.470370390278443, "output_per_gpu": 1821.9185318585464, "total_per_gpu": 16400.863632593417, "tpot_error_pct": 33.21, "tpot_ms": 54.14076593322311, "tpot_relative": 1.33207, "ttft_error_pct": 16.87, "ttft_ms": 74804.08272170436, "ttft_relative": 0.831338, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29358158271", "measured": {"e2e_ms": 127387.93269006108, "interactivity_tok_s": 24.60381718735145, "output_per_gpu": 3412.679193215791, "total_per_gpu": 15392.645332304815, "tpot_ms": 40.644099750265134, "tpot_relative": 1.0, "ttft_ms": 89980.35718914068, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 20.37, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.96, "tpot_shape_error_pct": null, "ttft_mape_pct": 23.89, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "9e873b33b1c39fd8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10662.296, "interactivity_tok_s": 101.5331505736623, "output_per_gpu": 74.80114999999999, "total_per_gpu": null, "tpot_error_pct": 34.94, "tpot_ms": 9.849, "tpot_relative": 1.349429, "ttft_error_pct": 20.37, "ttft_ms": 586.769, "ttft_relative": 1.203729, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8575.680930474999, "interactivity_tok_s": 111.42369966280475, "output_per_gpu": 82.3641316854898, "total_per_gpu": 743.3156729600206, "tpot_error_pct": 22.96, "tpot_ms": 8.97475135923725, "tpot_relative": 1.229647, "ttft_error_pct": 23.89, "ttft_ms": 371.0171292062508, "ttft_relative": 0.761124, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 7162.413700384786, "interactivity_tok_s": 137.01177363236926, "output_per_gpu": 123.66592115554445, "total_per_gpu": 893.6019145571812, "tpot_ms": 7.298642835492412, "tpot_relative": 1.0, "ttft_ms": 487.4594829088892, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.03, "tpot_shape_error_pct": 11.06, "ttft_mape_pct": 45.19, "ttft_shape_error_pct": 60.39}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.23, "tpot_shape_error_pct": 8.41, "ttft_mape_pct": 22.95, "ttft_shape_error_pct": 9.93}, "framework": "dynamo-vllm", "id": "cba741eaee47dbfe", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10891.448, "interactivity_tok_s": 99.27529038022436, "output_per_gpu": 81.22822222222221, "total_per_gpu": null, "tpot_error_pct": 31.15, "tpot_ms": 10.073, "tpot_relative": 1.311502, "ttft_error_pct": 2.12, "ttft_ms": 586.769, "ttft_relative": 1.021222, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8751.391697599991, "interactivity_tok_s": 110.53014688235228, "output_per_gpu": 89.46241647568645, "total_per_gpu": 807.375916511601, "tpot_error_pct": 17.8, "tpot_ms": 9.047305447484792, "tpot_relative": 1.177956, "ttft_error_pct": 16.75, "ttft_ms": 478.3366547000004, "ttft_relative": 0.832505, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 7599.088552383182, "interactivity_tok_s": 130.1996971671228, "output_per_gpu": 115.97038897529427, "total_per_gpu": 931.1049986925714, "tpot_ms": 7.680509415597271, "tpot_relative": 1.0, "ttft_ms": 574.5754061470507, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13145.117, "interactivity_tok_s": 81.45975887911372, "output_per_gpu": 133.35666666666668, "total_per_gpu": null, "tpot_error_pct": 42.6, "tpot_ms": 12.276, "tpot_relative": 1.598331, "ttft_error_pct": 29.14, "ttft_ms": 586.769, "ttft_relative": 1.021222, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10896.661467143746, "interactivity_tok_s": 90.23163638574282, "output_per_gpu": 145.92775627360365, "total_per_gpu": 1304.9324610155854, "tpot_error_pct": 28.74, "tpot_ms": 11.082587438899717, "tpot_relative": 1.44295, "ttft_error_pct": 21.14, "ttft_ms": 652.991199715623, "ttft_relative": 1.136476, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 8776.56941129535, "interactivity_tok_s": 116.163742882603, "output_per_gpu": 201.6618360041956, "total_per_gpu": 1609.483395902022, "tpot_ms": 8.608538044530956, "tpot_relative": 1.120829, "ttft_ms": 828.0853166033921, "ttft_relative": 1.441213, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15133.829, "interactivity_tok_s": 70.32348804500702, "output_per_gpu": 230.28622222222222, "total_per_gpu": null, "tpot_error_pct": 39.82, "tpot_ms": 14.22, "tpot_relative": 1.85144, "ttft_error_pct": 59.67, "ttft_ms": 586.769, "ttft_relative": 1.021222, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12792.976096701559, "interactivity_tok_s": 77.89514588634526, "output_per_gpu": 246.64320816138965, "total_per_gpu": 2223.187117635153, "tpot_error_pct": 26.23, "tpot_ms": 12.837770423577785, "tpot_relative": 1.671474, "ttft_error_pct": 33.03, "ttft_ms": 974.4227405468755, "ttft_relative": 1.695901, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 10815.230339042682, "interactivity_tok_s": 98.32437986889848, "output_per_gpu": 328.34599412152215, "total_per_gpu": 2639.2861713737752, "tpot_ms": 10.170417564121506, "tpot_relative": 1.324185, "ttft_ms": 1455.0158928330347, "ttft_relative": 2.532332, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18375.716, "interactivity_tok_s": 57.50761975961815, "output_per_gpu": 314.11199999999997, "total_per_gpu": null, "tpot_error_pct": 54.55, "tpot_ms": 17.389, "tpot_relative": 2.264043, "ttft_error_pct": 89.84, "ttft_ms": 586.769, "ttft_relative": 1.021222, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17819.45737136016, "interactivity_tok_s": 69.35813748151759, "output_per_gpu": 353.52121983443135, "total_per_gpu": 3191.618506668641, "tpot_error_pct": 28.15, "tpot_ms": 14.417918881781938, "tpot_relative": 1.877209, "ttft_error_pct": 20.88, "ttft_ms": 4568.896049663277, "ttft_relative": 7.951778, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 16115.875107064312, "interactivity_tok_s": 88.87911823075991, "output_per_gpu": 439.2423378664648, "total_per_gpu": 3528.903832966345, "tpot_ms": 11.251236734861225, "tpot_relative": 1.464908, "ttft_ms": 5774.931209980423, "ttft_relative": 10.05078, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 18}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 66.93, "tpot_shape_error_pct": 10.58, "ttft_mape_pct": 40.82, "ttft_shape_error_pct": 14.29}, "framework": "dynamo-vllm", "id": "c91d4d3b6dc9d44f", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 6367.600501699993, "interactivity_tok_s": 152.01962541815607, "output_per_gpu": 16.23322107583073, "total_per_gpu": 145.3120195969988, "tpot_error_pct": 53.91, "tpot_ms": 6.578098039969039, "tpot_relative": 1.539081, "ttft_error_pct": 33.18, "ttft_ms": 254.65506469999946, "ttft_relative": 0.668201, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 4352.910677893669, "interactivity_tok_s": 233.9704971062348, "output_per_gpu": 26.712041270538457, "total_per_gpu": 213.43854797520981, "tpot_ms": 4.274043148038224, "tpot_relative": 1.0, "ttft_ms": 381.1052327946527, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 6795.297121449998, "interactivity_tok_s": 143.196025549483, "output_per_gpu": 30.43195026079131, "total_per_gpu": 271.19333638177187, "tpot_error_pct": 58.16, "tpot_ms": 6.98343404548221, "tpot_relative": 1.633918, "ttft_error_pct": 43.52, "ttft_ms": 271.2676169999998, "ttft_relative": 0.711792, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 4594.812691450352, "interactivity_tok_s": 226.47583398871305, "output_per_gpu": 50.22285165972072, "total_per_gpu": 398.57867765872794, "tpot_ms": 4.415482139475585, "tpot_relative": 1.033093, "ttft_ms": 480.25581810070435, "ttft_relative": 1.260166, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7655.3584203750015, "interactivity_tok_s": 125.21693445968447, "output_per_gpu": 52.06183237143838, "total_per_gpu": 468.00482405249954, "tpot_error_pct": 68.47, "tpot_ms": 7.986140247842958, "tpot_relative": 1.868521, "ttft_error_pct": 45.01, "ttft_ms": 315.3340849999998, "ttft_relative": 0.82742, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 4934.590723601286, "interactivity_tok_s": 210.95139028828646, "output_per_gpu": 92.06924345023648, "total_per_gpu": 737.3602573371705, "tpot_ms": 4.740428582307036, "tpot_relative": 1.10912, "ttft_ms": 573.4574407768378, "ttft_relative": 1.504722, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9448.500963749964, "interactivity_tok_s": 102.28082741984657, "output_per_gpu": 85.5936073292605, "total_per_gpu": 761.3947104711619, "tpot_error_pct": 81.11, "tpot_ms": 9.777003425042297, "tpot_relative": 2.28753, "ttft_error_pct": 41.29, "ttft_ms": 404.90118812499935, "ttft_relative": 1.062439, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 5684.748868126553, "interactivity_tok_s": 185.24042027792518, "output_per_gpu": 157.7908117851416, "total_per_gpu": 1250.9806791701426, "tpot_ms": 5.398389824961806, "tpot_relative": 1.263064, "ttft_ms": 689.6290303237038, "ttft_relative": 1.80955, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11600.749625012442, "interactivity_tok_s": 82.65311133168663, "output_per_gpu": 135.6727183459814, "total_per_gpu": 1224.4123245874632, "tpot_error_pct": 73.02, "tpot_ms": 12.098758097405476, "tpot_relative": 2.830752, "ttft_error_pct": 41.09, "ttft_ms": 539.6936245249977, "ttft_relative": 1.416128, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 7311.6067763443425, "interactivity_tok_s": 143.00973817543095, "output_per_gpu": 240.89659050403318, "total_per_gpu": 1934.1145750066119, "tpot_ms": 6.992530807750264, "tpot_relative": 1.636046, "ttft_ms": 916.08548182503, "ttft_relative": 2.40376, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 9}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 115.25, "tpot_shape_error_pct": null, "ttft_mape_pct": 26.75, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "ec9978af315e6d1e", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 244476.36391343517, "interactivity_tok_s": 18.200833102255128, "output_per_gpu": 1846.6361402877233, "total_per_gpu": 16623.370906152588, "tpot_error_pct": 115.25, "tpot_ms": 54.94253995857461, "tpot_relative": 2.152457, "ttft_error_pct": 26.75, "ttft_ms": 193908.06697202352, "ttft_relative": 0.732487, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30828231519", "measured": {"e2e_ms": 288219.5539763401, "interactivity_tok_s": 39.17651825477151, "output_per_gpu": 3101.0797438755535, "total_per_gpu": 13987.198310220052, "tpot_ms": 25.5254944683147, "tpot_relative": 1.0, "ttft_ms": 264725.589539839, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 145.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 99.44, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.56, "tpot_shape_error_pct": null, "ttft_mape_pct": 1.19, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "8430d68aa0a8fa09", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 57375.749, "interactivity_tok_s": 18.014123072488832, "output_per_gpu": 942.0287916666666, "total_per_gpu": null, "tpot_error_pct": 145.23, "tpot_ms": 55.512, "tpot_relative": 2.452281, "ttft_error_pct": 99.44, "ttft_ms": 586.973, "ttft_relative": 0.005593, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 121505.44874417882, "interactivity_tok_s": 51.70220472282609, "output_per_gpu": 1231.5376277851115, "total_per_gpu": 11086.269961317565, "tpot_error_pct": 14.56, "tpot_ms": 19.34153495699011, "tpot_relative": 0.854426, "ttft_error_pct": 1.19, "ttft_ms": 103703.45886469031, "ttft_relative": 0.988114, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 125786.26457421322, "interactivity_tok_s": 44.17568936792968, "output_per_gpu": 1775.7924932504613, "total_per_gpu": 10679.446649702611, "tpot_ms": 22.636884999602795, "tpot_relative": 1.0, "ttft_ms": 104950.91156807766, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 61.71, "tpot_shape_error_pct": null, "ttft_mape_pct": 82.4, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "27ef55580b9ca65d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 29362.69424302294, "interactivity_tok_s": 32.96621220533299, "output_per_gpu": 1109.2104130053567, "total_per_gpu": 9980.023482253278, "tpot_error_pct": 61.71, "tpot_ms": 30.334088544095117, "tpot_relative": 1.617062, "ttft_error_pct": 82.4, "ttft_ms": 1425.0124500628863, "ttft_relative": 0.176046, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 25380.296801305565, "interactivity_tok_s": 53.308401448952374, "output_per_gpu": 2243.8160636855614, "total_per_gpu": 11549.230169267526, "tpot_ms": 18.75876921497244, "tpot_relative": 1.0, "ttft_ms": 8094.530300916972, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 7}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 77.26, "tpot_shape_error_pct": null, "ttft_mape_pct": 80.67, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "430db7bbcb656f48", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24010.672940407247, "interactivity_tok_s": 40.26734535080636, "output_per_gpu": 790.846423651553, "total_per_gpu": 7139.826239684103, "tpot_error_pct": 77.26, "tpot_ms": 24.834018515203034, "tpot_relative": 1.772648, "ttft_error_pct": 80.67, "ttft_ms": 1188.6715082179712, "ttft_relative": 0.193256, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 19026.910235633088, "interactivity_tok_s": 71.37983571825994, "output_per_gpu": 1486.5483693402023, "total_per_gpu": 8957.275607701082, "tpot_ms": 14.009558721135951, "tpot_relative": 1.0, "ttft_ms": 6150.757621265143, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.01, "tpot_shape_error_pct": null, "ttft_mape_pct": 4.74, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.7, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.79, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "e2bc8e6364439980", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7728.332, "interactivity_tok_s": 143.2459533018192, "output_per_gpu": 15.516617647058824, "total_per_gpu": null, "tpot_error_pct": 3.01, "tpot_ms": 6.981, "tpot_relative": 1.030123, "ttft_error_pct": 4.74, "ttft_ms": 586.769, "ttft_relative": 1.047356, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6250.978961150005, "interactivity_tok_s": 154.8368797636787, "output_per_gpu": 16.812803066337786, "total_per_gpu": 151.1370726399453, "tpot_error_pct": 4.7, "tpot_ms": 6.458409660064578, "tpot_relative": 0.953009, "ttft_error_pct": 43.79, "ttft_ms": 314.90329935000034, "ttft_relative": 0.562088, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6811.9427564844955, "interactivity_tok_s": 147.56098886127538, "output_per_gpu": 16.71751407572084, "total_per_gpu": 141.7621962247751, "tpot_ms": 6.776858895545335, "tpot_relative": 1.0, "ttft_ms": 560.2385645091999, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 34}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 62.63, "tpot_shape_error_pct": 8.1, "ttft_mape_pct": 47.75, "ttft_shape_error_pct": 30.76}, "framework": "dynamo-vllm", "id": "a5b8e6168256d51c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 10037.939123125976, "interactivity_tok_s": 95.97073695173667, "output_per_gpu": 145.09851652896782, "total_per_gpu": 1290.7183892529029, "tpot_error_pct": 62.55, "tpot_ms": 10.419842878802696, "tpot_relative": 1.625462, "ttft_error_pct": 30.92, "ttft_ms": 399.66079112500324, "ttft_relative": 0.690778, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 6509.898457361851, "interactivity_tok_s": 155.99681814422885, "output_per_gpu": 276.57736947602643, "total_per_gpu": 1973.4587041937782, "tpot_ms": 6.410387159790895, "tpot_relative": 1.0, "ttft_ms": 578.5660167137394, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 11847.547369037453, "interactivity_tok_s": 80.79234753254025, "output_per_gpu": 239.06817837506475, "total_per_gpu": 2157.5304717683766, "tpot_error_pct": 46.64, "tpot_ms": 12.377409872850098, "tpot_relative": 1.930837, "ttft_error_pct": 31.06, "ttft_ms": 532.0270296812481, "ttft_relative": 0.919561, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 8490.547671569584, "interactivity_tok_s": 118.4700445663327, "output_per_gpu": 415.5208728214155, "total_per_gpu": 3002.526840233224, "tpot_ms": 8.44095234082645, "tpot_relative": 1.316762, "ttft_ms": 771.6707637006039, "ttft_relative": 1.333764, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13641.497571724894, "interactivity_tok_s": 70.99641533084957, "output_per_gpu": 314.3210877742569, "total_per_gpu": 2828.2270441770806, "tpot_error_pct": 49.49, "tpot_ms": 14.085218180944935, "tpot_relative": 2.197249, "ttft_error_pct": 28.3, "ttft_ms": 715.9370690499965, "ttft_relative": 1.237434, "unavailable_metrics": {}}, "concurrency": 24, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 9640.540109592015, "interactivity_tok_s": 106.12950277013542, "output_per_gpu": 554.9325011028491, "total_per_gpu": 4006.3408781357493, "tpot_ms": 9.422450627756993, "tpot_relative": 1.469872, "ttft_ms": 998.5281001496332, "ttft_relative": 1.725867, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15619.273690212533, "interactivity_tok_s": 62.38211579059035, "output_per_gpu": 367.46322480156704, "total_per_gpu": 3285.9731590334195, "tpot_error_pct": 58.85, "tpot_ms": 16.030235386002072, "tpot_relative": 2.500666, "ttft_error_pct": 41.31, "ttft_ms": 804.2430993531244, "ttft_relative": 1.390063, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 10687.214900508843, "interactivity_tok_s": 99.09424439104843, "output_per_gpu": 668.24903126644, "total_per_gpu": 4800.02645674471, "tpot_ms": 10.091403452795628, "tpot_relative": 1.574227, "ttft_ms": 1370.3085493058097, "ttft_relative": 2.368457, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 19436.24002151664, "interactivity_tok_s": 50.25337469727335, "output_per_gpu": 445.8013304945819, "total_per_gpu": 3996.140710022855, "tpot_error_pct": 74.05, "tpot_ms": 19.89916112149694, "tpot_relative": 3.104206, "ttft_error_pct": 71.98, "ttft_ms": 1026.4000512541677, "ttft_relative": 1.774041, "unavailable_metrics": {}}, "concurrency": 48, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 14234.094781733136, "interactivity_tok_s": 87.4662183363022, "output_per_gpu": 755.7678872106202, "total_per_gpu": 5438.253980889827, "tpot_ms": 11.432985431644727, "tpot_relative": 1.783509, "ttft_ms": 3663.522372737134, "ttft_relative": 6.332073, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 20877.60520684839, "interactivity_tok_s": 47.211817863706486, "output_per_gpu": 548.0423006785019, "total_per_gpu": 4939.931619727888, "tpot_error_pct": 84.21, "tpot_ms": 21.181137377231515, "tpot_relative": 3.30419, "ttft_error_pct": 82.95, "ttft_ms": 1380.4449429624988, "ttft_relative": 2.385977, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 18677.579748082415, "interactivity_tok_s": 86.96769288262115, "output_per_gpu": 760.1754793482759, "total_per_gpu": 5499.347029484115, "tpot_ms": 11.498522806045727, "tpot_relative": 1.793733, "ttft_ms": 8097.230985193846, "ttft_relative": 13.995345, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 5}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.02, "tpot_shape_error_pct": 25.0, "ttft_mape_pct": 38.94, "ttft_shape_error_pct": 40.27}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.89, "tpot_shape_error_pct": 22.64, "ttft_mape_pct": 21.41, "ttft_shape_error_pct": 18.91}, "framework": "dynamo-vllm", "id": "421f79c4f024661c", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7726.286, "interactivity_tok_s": 143.2870038687491, "output_per_gpu": 21.98183333333333, "total_per_gpu": null, "tpot_error_pct": 6.13, "tpot_ms": 6.979, "tpot_relative": 1.061254, "ttft_error_pct": 94.28, "ttft_ms": 586.769, "ttft_relative": 1.942833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6288.414777199998, "interactivity_tok_s": 154.57790630217445, "output_per_gpu": 24.656452459555812, "total_per_gpu": 220.7127523403171, "tpot_error_pct": 1.63, "tpot_ms": 6.469229813768884, "tpot_relative": 0.983736, "ttft_error_pct": 8.42, "ttft_ms": 276.57473769999945, "ttft_relative": 0.915758, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 6413.235774566419, "interactivity_tok_s": 152.0639142283306, "output_per_gpu": 36.2633314420928, "total_per_gpu": 217.31752160066307, "tpot_ms": 6.57618216047271, "tpot_relative": 1.0, "ttft_ms": 302.01718690805137, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10055.657, "interactivity_tok_s": 108.038029386344, "output_per_gpu": 66.25966666666666, "total_per_gpu": null, "tpot_error_pct": 26.47, "tpot_ms": 9.256, "tpot_relative": 1.407504, "ttft_error_pct": 42.65, "ttft_ms": 586.769, "ttft_relative": 1.942833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8058.179734124997, "interactivity_tok_s": 118.75135388139844, "output_per_gpu": 74.2402357585509, "total_per_gpu": 667.3754436053622, "tpot_error_pct": 15.06, "tpot_ms": 8.420956623355542, "tpot_relative": 1.280524, "ttft_error_pct": 22.61, "ttft_ms": 318.3282605500012, "ttft_relative": 1.054007, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 7146.594864601502, "interactivity_tok_s": 136.63292921215347, "output_per_gpu": 127.28850214459305, "total_per_gpu": 764.5670734885276, "tpot_ms": 7.318879905204069, "tpot_relative": 1.112938, "ttft_ms": 411.32160523557104, "ttft_relative": 1.361915, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12204.98, "interactivity_tok_s": 88.0514220304658, "output_per_gpu": 108.18216666666666, "total_per_gpu": null, "tpot_error_pct": 36.66, "tpot_ms": 11.357, "tpot_relative": 1.72699, "ttft_error_pct": 12.13, "ttft_ms": 586.769, "ttft_relative": 1.942833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9944.502305150001, "interactivity_tok_s": 96.69336417555374, "output_per_gpu": 122.05635944861153, "total_per_gpu": 1085.7477487313488, "tpot_error_pct": 24.45, "tpot_ms": 10.341971328915895, "tpot_relative": 1.572641, "ttft_error_pct": 27.7, "ttft_ms": 378.3125285999998, "ttft_relative": 1.252619, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 8215.385781732039, "interactivity_tok_s": 120.33235570438366, "output_per_gpu": 219.78907040270897, "total_per_gpu": 1306.8816117487934, "tpot_ms": 8.310316823321115, "tpot_relative": 1.263699, "ttft_ms": 523.2798537617782, "ttft_relative": 1.732616, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14641.766, "interactivity_tok_s": 72.78550112817527, "output_per_gpu": 178.68116666666666, "total_per_gpu": null, "tpot_error_pct": 34.83, "tpot_ms": 13.739, "tpot_relative": 2.089206, "ttft_error_pct": 6.67, "ttft_ms": 586.769, "ttft_relative": 1.942833, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11863.247113506231, "interactivity_tok_s": 80.16561338693862, "output_per_gpu": 199.62673620757354, "total_per_gpu": 1801.5813282845102, "tpot_error_pct": 22.42, "tpot_ms": 12.474176367531294, "tpot_relative": 1.896872, "ttft_error_pct": 26.91, "ttft_ms": 459.5623264562496, "ttft_relative": 1.521643, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 9948.518952314043, "interactivity_tok_s": 98.13858022620767, "output_per_gpu": 357.994190102201, "total_per_gpu": 2155.702305901136, "tpot_ms": 10.189672580294294, "tpot_relative": 1.549481, "ttft_ms": 628.7255562783685, "ttft_relative": 2.081754, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 6}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.99, "tpot_shape_error_pct": 15.37, "ttft_mape_pct": 84.36, "ttft_shape_error_pct": 86.11}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.56, "tpot_shape_error_pct": 2.6, "ttft_mape_pct": 41.01, "ttft_shape_error_pct": 16.41}, "framework": "dynamo-vllm", "id": "bdb4a056c1e05a4a", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 18389.015, "interactivity_tok_s": 57.46465923457074, "output_per_gpu": 565.4015999999999, "total_per_gpu": null, "tpot_error_pct": 9.0, "tpot_ms": 17.402, "tpot_relative": 0.910024, "ttft_error_pct": 72.54, "ttft_ms": 586.769, "ttft_relative": 0.274643, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17962.63878479062, "interactivity_tok_s": 62.73774804239443, "output_per_gpu": 627.3282637092667, "total_per_gpu": 5644.331075054039, "tpot_error_pct": 16.65, "tpot_ms": 15.939367146622152, "tpot_relative": 0.833536, "ttft_error_pct": 53.61, "ttft_ms": 3281.8663667113206, "ttft_relative": 1.536113, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 19757.289334980032, "interactivity_tok_s": 52.29419337163821, "output_per_gpu": 710.46719824102, "total_per_gpu": 5119.621463718174, "tpot_ms": 19.12258198330583, "tpot_relative": 1.0, "ttft_ms": 2136.475167264689, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20645.753, "interactivity_tok_s": 50.99959200326397, "output_per_gpu": 565.4015999999999, "total_per_gpu": null, "tpot_error_pct": 4.99, "tpot_ms": 19.608, "tpot_relative": 1.025385, "ttft_error_pct": 96.18, "ttft_ms": 586.769, "ttft_relative": 0.274643, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34461.67333242305, "interactivity_tok_s": 62.6087180243347, "output_per_gpu": 651.2252274096971, "total_per_gpu": 5861.07435760226, "tpot_error_pct": 14.48, "tpot_ms": 15.972216514820204, "tpot_relative": 0.835254, "ttft_error_pct": 28.41, "ttft_ms": 19749.301242676564, "ttft_relative": 9.243871, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28214046079", "measured": {"e2e_ms": 32581.441863249074, "interactivity_tok_s": 53.54424082157267, "output_per_gpu": 857.1557687028954, "total_per_gpu": 6182.817183053473, "tpot_ms": 18.676144897307157, "tpot_relative": 0.976654, "ttft_ms": 15380.067609536354, "ttft_relative": 7.198805, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.96, "tpot_shape_error_pct": 0.35, "ttft_mape_pct": 93.75, "ttft_shape_error_pct": 58.02}, "framework": "dynamo-vllm", "id": "2fd3351125ec705f", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24218.107665403302, "interactivity_tok_s": 52.05944680536266, "output_per_gpu": 1165.2207934755309, "total_per_gpu": 10491.719733630542, "tpot_error_pct": 45.21, "tpot_ms": 19.208809569927848, "tpot_relative": 1.452119, "ttft_error_pct": 95.15, "ttft_ms": 6531.277183010814, "ttft_relative": 0.048473, "unavailable_metrics": {}}, "concurrency": 768, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 146920.0512669791, "interactivity_tok_s": 75.59649716616393, "output_per_gpu": 181.23357779563094, "total_per_gpu": 1089.9653713813598, "tpot_ms": 13.228126136611365, "tpot_relative": 1.0, "ttft_ms": 134741.5987058187, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 31700.36997857395, "interactivity_tok_s": 51.9518621281403, "output_per_gpu": 1184.0083047005476, "total_per_gpu": 10653.889342948907, "tpot_error_pct": 44.7, "tpot_ms": 19.248588193691308, "tpot_relative": 1.455126, "ttft_error_pct": 92.34, "ttft_ms": 13975.840550652767, "ttft_relative": 0.103723, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 194712.7113346532, "interactivity_tok_s": 75.17401229444775, "output_per_gpu": 181.33170187713608, "total_per_gpu": 1090.0866764730192, "tpot_ms": 13.30246942365026, "tpot_relative": 1.00562, "ttft_ms": 182464.6433432158, "ttft_relative": 1.354182, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.19, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.37, "ttft_shape_error_pct": null}, "framework": "dynamo-vllm", "id": "1ab5ee12586df023", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 25554.052722408156, "interactivity_tok_s": 57.954582762803525, "output_per_gpu": 839.8918716934178, "total_per_gpu": 7559.087862614267, "tpot_error_pct": 24.19, "tpot_ms": 17.25489085294254, "tpot_relative": 1.24186, "ttft_error_pct": 43.37, "ttft_ms": 9660.362080912386, "ttft_relative": 0.566332, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "verified", "infx_run_id": "30828231519", "measured": {"e2e_ms": 29854.987134295443, "interactivity_tok_s": 71.97147171590605, "output_per_gpu": 938.2175218041597, "total_per_gpu": 6445.266706535731, "tpot_ms": 13.894394211463583, "tpot_relative": 1.0, "ttft_ms": 17057.78323047657, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 21}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.51, "tpot_shape_error_pct": 13.47, "ttft_mape_pct": 38.79, "ttft_shape_error_pct": 47.86}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.07, "tpot_shape_error_pct": 11.71, "ttft_mape_pct": 5.28, "ttft_shape_error_pct": 2.08}, "framework": "vllm", "id": "1ebcf67a26afdc24", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 11922.912946148272, "interactivity_tok_s": 87.99067135466343, "output_per_gpu": 171.60236003073106, "total_per_gpu": null, "tpot_error_pct": 57.28, "tpot_ms": 11.364841120137685, "tpot_relative": 1.57281, "ttft_error_pct": 32.59, "ttft_ms": 296.6804802474206, "ttft_relative": 1.325887, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10429.6671233006, "interactivity_tok_s": 89.92747832513585, "output_per_gpu": 172.75842131440925, "total_per_gpu": 1552.9951768503972, "tpot_error_pct": 53.89, "tpot_ms": 11.120071624653658, "tpot_relative": 1.538935, "ttft_error_pct": 6.32, "ttft_ms": 209.6105680250008, "ttft_relative": 0.936765, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 6866.135041225061, "interactivity_tok_s": 138.39258483323772, "output_per_gpu": 262.0715822837774, "total_per_gpu": 2355.8672288139974, "tpot_ms": 7.225820669546669, "tpot_relative": 1.0, "ttft_ms": 223.76007969942293, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16341.39116711995, "interactivity_tok_s": 63.879188322920406, "output_per_gpu": 250.4070772281247, "total_per_gpu": null, "tpot_error_pct": 72.65, "tpot_ms": 15.654550820915665, "tpot_relative": 2.166474, "ttft_error_pct": 22.64, "ttft_ms": 326.78567732322574, "ttft_relative": 1.460429, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14476.862806599986, "interactivity_tok_s": 65.06591518033174, "output_per_gpu": 252.30933908284993, "total_per_gpu": 2244.408223632424, "tpot_error_pct": 69.5, "tpot_ms": 15.369029963360632, "tpot_relative": 2.12696, "ttft_error_pct": 7.02, "ttft_ms": 247.74651352499973, "ttft_relative": 1.107197, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 8664.700119049667, "interactivity_tok_s": 110.28732749123672, "output_per_gpu": 421.1021772821001, "total_per_gpu": 3745.898558955508, "tpot_ms": 9.067224882019728, "tpot_relative": 1.254837, "ttft_ms": 266.4495362256275, "ttft_relative": 1.190782, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21730.977568080416, "interactivity_tok_s": 47.86314261142747, "output_per_gpu": 376.6052389663815, "total_per_gpu": null, "tpot_error_pct": 56.29, "tpot_ms": 20.892903086586024, "tpot_relative": 2.891423, "ttft_error_pct": 4.25, "ttft_ms": 357.53771050291647, "ttft_relative": 1.597862, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19209.243779162476, "interactivity_tok_s": 48.43404462450803, "output_per_gpu": 372.12848293366204, "total_per_gpu": 3358.3664158042366, "tpot_error_pct": 54.44, "tpot_ms": 20.64663415481084, "tpot_relative": 2.857341, "ttft_error_pct": 6.02, "ttft_ms": 322.30867136250004, "ttft_relative": 1.440421, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 12574.258844891665, "interactivity_tok_s": 74.8036923066158, "output_per_gpu": 568.7513917180912, "total_per_gpu": 5132.838953444097, "tpot_ms": 13.368324064821033, "tpot_relative": 1.850077, "ttft_ms": 342.9480864180732, "ttft_relative": 1.53266, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28082.897517690693, "interactivity_tok_s": 36.93969220215954, "output_per_gpu": 582.8458402374275, "total_per_gpu": null, "tpot_error_pct": 43.95, "tpot_ms": 27.071151392580873, "tpot_relative": 3.746447, "ttft_error_pct": 20.38, "ttft_ms": 389.1096430804595, "ttft_relative": 1.738959, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25395.827575818512, "interactivity_tok_s": 37.09651135211491, "output_per_gpu": 570.3774325825194, "total_per_gpu": 5100.496614311974, "tpot_error_pct": 43.35, "tpot_ms": 26.95671273527959, "tpot_relative": 3.730609, "ttft_error_pct": 7.45, "ttft_ms": 452.2820058468757, "ttft_relative": 2.021281, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 17893.855542181427, "interactivity_tok_s": 53.176008496682066, "output_per_gpu": 810.876480109353, "total_per_gpu": 7251.115673873734, "tpot_ms": 18.805473149839656, "tpot_relative": 2.602538, "ttft_ms": 488.694582962853, "ttft_relative": 2.184011, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 40593.476670102544, "interactivity_tok_s": 25.466059371567027, "output_per_gpu": 806.4349911696614, "total_per_gpu": null, "tpot_error_pct": 36.17, "tpot_ms": 39.2679521165534, "tpot_relative": 5.434393, "ttft_error_pct": 46.95, "ttft_ms": 422.3616548684161, "ttft_relative": 1.887565, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37257.72336536554, "interactivity_tok_s": 25.244555590897434, "output_per_gpu": 778.293969338918, "total_per_gpu": 7015.369039617759, "tpot_error_pct": 37.37, "tpot_ms": 39.61250165008155, "tpot_relative": 5.482076, "ttft_error_pct": 6.69, "ttft_ms": 742.9507656625012, "ttft_relative": 3.320301, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 27385.690627037548, "interactivity_tok_s": 34.677526338182325, "output_per_gpu": 1060.6571236287605, "total_per_gpu": 9560.527820966574, "tpot_ms": 28.837120336894728, "tpot_relative": 3.990844, "ttft_ms": 796.2128823258354, "ttft_relative": 3.558333, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 56834.31680401072, "interactivity_tok_s": 18.146598011094948, "output_per_gpu": 1151.9800655962092, "total_per_gpu": null, "tpot_error_pct": 23.55, "tpot_ms": 55.106747798600786, "tpot_relative": 7.626365, "ttft_error_pct": 65.44, "ttft_ms": 460.11380604211536, "ttft_relative": 2.056282, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 53005.544566948585, "interactivity_tok_s": 17.802158897256277, "output_per_gpu": 1093.5927983630024, "total_per_gpu": 9873.045288906767, "tpot_error_pct": 25.94, "tpot_ms": 56.17296226662279, "tpot_relative": 7.773921, "ttft_error_pct": 2.64, "ttft_ms": 1296.160664826564, "ttft_relative": 5.792636, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 42398.88458690289, "interactivity_tok_s": 22.41979263183386, "output_per_gpu": 1370.6765048145064, "total_per_gpu": 12374.579668713268, "tpot_ms": 44.6034455546257, "tpot_relative": 6.172786, "ttft_ms": 1331.3226433835553, "ttft_relative": 5.949777, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 86720.06969319722, "interactivity_tok_s": 11.865984822551823, "output_per_gpu": 1509.961886138474, "total_per_gpu": null, "tpot_error_pct": 14.69, "tpot_ms": 84.27450523107498, "tpot_relative": 11.662967, "ttft_error_pct": 79.3, "ttft_ms": 507.25084180751196, "ttft_relative": 2.266941, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 81761.37799904362, "interactivity_tok_s": 11.63443230461449, "output_per_gpu": 1425.4485224157222, "total_per_gpu": 12825.348157260263, "tpot_error_pct": 16.97, "tpot_ms": 85.95176574308455, "tpot_relative": 11.895087, "ttft_error_pct": 0.78, "ttft_ms": 2431.787252305464, "ttft_relative": 10.867833, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 70277.0817147835, "interactivity_tok_s": 13.608726574384134, "output_per_gpu": 1661.5075015014545, "total_per_gpu": 14949.268133894098, "tpot_ms": 73.4822611457498, "tpot_relative": 10.1694, "ttft_ms": 2451.0108587795003, "ttft_relative": 10.953745, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.96, "tpot_shape_error_pct": 12.14, "ttft_mape_pct": 36.3, "ttft_shape_error_pct": 45.93}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.79, "tpot_shape_error_pct": 12.23, "ttft_mape_pct": 26.08, "ttft_shape_error_pct": 69.4}, "framework": "vllm", "id": "073de936bb3af4c0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6770.220720650413, "interactivity_tok_s": 154.69188426539804, "output_per_gpu": 18.887862785619063, "total_per_gpu": null, "tpot_error_pct": 20.75, "tpot_ms": 6.4644632441372565, "tpot_relative": 1.207545, "ttft_error_pct": 28.82, "ttft_ms": 157.07482189799921, "ttft_relative": 0.711839, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6102.100235700001, "interactivity_tok_s": 154.83928689044546, "output_per_gpu": 19.05696325990622, "total_per_gpu": 170.58880709792007, "tpot_error_pct": 20.64, "tpot_ms": 6.458309257827679, "tpot_relative": 1.206396, "ttft_error_pct": 54.5, "ttft_ms": 100.39760629999941, "ttft_relative": 0.454987, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 5195.556564070284, "interactivity_tok_s": 186.7974807028254, "output_per_gpu": 22.38080546855596, "total_per_gpu": 200.34225047828292, "tpot_ms": 5.353391256870814, "tpot_relative": 1.0, "ttft_ms": 220.6605496816337, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6974.684772541924, "interactivity_tok_s": 150.53645275394433, "output_per_gpu": 36.668323851257284, "total_per_gpu": null, "tpot_error_pct": 14.99, "tpot_ms": 6.642909286792651, "tpot_relative": 1.240879, "ttft_error_pct": 5.93, "ttft_ms": 178.98857215304196, "ttft_relative": 0.811149, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6219.462866650004, "interactivity_tok_s": 153.0378301663568, "output_per_gpu": 37.474765046173516, "total_per_gpu": 333.9551516055396, "tpot_error_pct": 13.11, "tpot_ms": 6.534332059680731, "tpot_relative": 1.220597, "ttft_error_pct": 40.23, "ttft_ms": 113.72398035000052, "ttft_relative": 0.51538, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 5586.224119283725, "interactivity_tok_s": 173.10495205378615, "output_per_gpu": 41.72854434293967, "total_per_gpu": 371.86256765465225, "tpot_ms": 5.776842245906899, "tpot_relative": 1.0791, "ttft_ms": 190.27140924008563, "ttft_relative": 0.862281, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8471.098272226205, "interactivity_tok_s": 123.66834271982532, "output_per_gpu": 60.38178091700709, "total_per_gpu": null, "tpot_error_pct": 25.7, "tpot_ms": 8.086143777842425, "tpot_relative": 1.510471, "ttft_error_pct": 21.54, "ttft_ms": 198.9731874934036, "ttft_relative": 0.901716, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7422.672936700006, "interactivity_tok_s": 125.9235322280104, "output_per_gpu": 60.557739657690945, "total_per_gpu": 544.377963712684, "tpot_error_pct": 23.45, "tpot_ms": 7.941327425514833, "tpot_relative": 1.48342, "ttft_error_pct": 24.31, "ttft_ms": 123.9001341500001, "ttft_relative": 0.561497, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 6076.412955898559, "interactivity_tok_s": 155.4553062939164, "output_per_gpu": 73.87028212475921, "total_per_gpu": 664.0497810728763, "tpot_ms": 6.43271705443955, "tpot_relative": 1.201615, "ttft_ms": 163.70335913961753, "ttft_relative": 0.741879, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10365.230273214924, "interactivity_tok_s": 100.82579987308557, "output_per_gpu": 98.69534713990507, "total_per_gpu": null, "tpot_error_pct": 0.06, "tpot_ms": 9.918096372741397, "tpot_relative": 1.852675, "ttft_error_pct": 1.23, "ttft_ms": 219.0176839004736, "ttft_relative": 0.992555, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9189.722373737492, "interactivity_tok_s": 102.36376375432339, "output_per_gpu": 99.15879524744591, "total_per_gpu": 882.0633287211474, "tpot_error_pct": 1.44, "tpot_ms": 9.769081981002916, "tpot_relative": 1.82484, "ttft_error_pct": 33.79, "ttft_ms": 146.8113022874997, "ttft_relative": 0.665326, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 9394.562048303487, "interactivity_tok_s": 100.88911138013076, "output_per_gpu": 96.61941001954033, "total_per_gpu": 859.4743230616681, "tpot_ms": 9.911872414379708, "tpot_relative": 1.851513, "ttft_ms": 221.73597595101455, "ttft_relative": 1.004874, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13723.868525828344, "interactivity_tok_s": 75.86440992328657, "output_per_gpu": 149.08332851990127, "total_per_gpu": null, "tpot_error_pct": 33.7, "tpot_ms": 13.18141142877393, "tpot_relative": 2.462254, "ttft_error_pct": 5.54, "ttft_ms": 239.28463419261192, "ttft_relative": 1.084402, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12093.700719093795, "interactivity_tok_s": 76.85874337274532, "output_per_gpu": 147.25476411381462, "total_per_gpu": 1328.9373886899405, "tpot_error_pct": 31.97, "tpot_ms": 13.010881470573292, "tpot_relative": 2.4304, "ttft_error_pct": 24.22, "ttft_ms": 191.97510395624994, "ttft_relative": 0.870002, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 9271.798546091304, "interactivity_tok_s": 101.43024829171533, "output_per_gpu": 191.6529612369214, "total_per_gpu": 1729.6200049869537, "tpot_ms": 9.858991936251412, "tpot_relative": 1.841635, "ttft_ms": 253.32282756280622, "ttft_relative": 1.14802, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17677.60240603631, "interactivity_tok_s": 58.73641631309628, "output_per_gpu": 231.479354836192, "total_per_gpu": null, "tpot_error_pct": 27.68, "tpot_ms": 17.025213024054263, "tpot_relative": 3.180267, "ttft_error_pct": 21.7, "ttft_ms": 260.8094824287984, "ttft_relative": 1.181949, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15896.14888531558, "interactivity_tok_s": 59.203587700518476, "output_per_gpu": 227.31965052250902, "total_per_gpu": 2032.7646951370148, "tpot_error_pct": 26.67, "tpot_ms": 16.890868253770414, "tpot_relative": 3.155172, "ttft_error_pct": 19.6, "ttft_ms": 267.77306626562483, "ttft_relative": 1.213507, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 12675.244871219184, "interactivity_tok_s": 74.99611210533519, "output_per_gpu": 284.7963201966882, "total_per_gpu": 2546.739376336671, "tpot_ms": 13.334024550438802, "tpot_relative": 2.490762, "ttft_ms": 333.070834215323, "ttft_relative": 1.509426, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23188.50848643661, "interactivity_tok_s": 44.66240641680792, "output_per_gpu": 352.9334370421872, "total_per_gpu": null, "tpot_error_pct": 14.48, "tpot_ms": 22.390195249839188, "tpot_relative": 4.182432, "ttft_error_pct": 45.22, "ttft_ms": 283.3387458511221, "ttft_relative": 1.284048, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21096.156666228195, "interactivity_tok_s": 44.594220724025774, "output_per_gpu": 342.7829451434127, "total_per_gpu": 3089.7693614543628, "tpot_error_pct": 14.65, "tpot_ms": 22.42443042538101, "tpot_relative": 4.188827, "ttft_error_pct": 17.97, "ttft_ms": 424.2923793468746, "ttft_relative": 1.922828, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 18551.170456263935, "interactivity_tok_s": 51.127922058373095, "output_per_gpu": 390.6739760222883, "total_per_gpu": 3521.448480834431, "tpot_ms": 19.558784314729106, "tpot_relative": 3.653532, "ttft_ms": 517.2706682184071, "ttft_relative": 2.344192, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31884.601293374308, "interactivity_tok_s": 32.39737540407204, "output_per_gpu": 513.3512522046593, "total_per_gpu": null, "tpot_error_pct": 0.05, "tpot_ms": 30.866697920051564, "tpot_relative": 5.765821, "ttft_error_pct": 65.26, "ttft_ms": 307.9693211615559, "ttft_relative": 1.39567, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 29260.71771112967, "interactivity_tok_s": 32.275084257074745, "output_per_gpu": 494.36969599013435, "total_per_gpu": 4463.210077169428, "tpot_error_pct": 0.43, "tpot_ms": 30.98365265400658, "tpot_relative": 5.787668, "ttft_error_pct": 16.97, "ttft_ms": 736.1226918257805, "ttft_relative": 3.335996, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 29293.30436670989, "interactivity_tok_s": 32.41345569609594, "output_per_gpu": 495.56758863108064, "total_per_gpu": 4474.024749973615, "tpot_ms": 30.85138497344625, "tpot_relative": 5.762961, "ttft_ms": 886.5294873372477, "ttft_relative": 4.017617, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 49204.88485715159, "interactivity_tok_s": 20.934769761590225, "output_per_gpu": 665.2997988926712, "total_per_gpu": null, "tpot_error_pct": 4.69, "tpot_ms": 47.767422875351414, "tpot_relative": 8.922834, "ttft_error_pct": 79.09, "ttft_ms": 338.81125566709613, "ttft_relative": 1.535441, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 45104.895472257114, "interactivity_tok_s": 21.10825744136804, "output_per_gpu": 645.491299960902, "total_per_gpu": 5807.751401959555, "tpot_error_pct": 5.47, "tpot_ms": 47.37482488915435, "tpot_relative": 8.849498, "ttft_error_pct": 15.11, "ttft_ms": 1375.4600693746083, "ttft_relative": 6.233376, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 47883.337044991735, "interactivity_tok_s": 19.953066414747415, "output_per_gpu": 609.4981628170038, "total_per_gpu": 5483.906304866753, "tpot_ms": 50.11760995597623, "tpot_relative": 9.361843, "ttft_ms": 1620.2790256710614, "ttft_relative": 7.342858, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 84269.86528529308, "interactivity_tok_s": 12.191463192283166, "output_per_gpu": 776.9325342854949, "total_per_gpu": null, "tpot_error_pct": 7.51, "tpot_ms": 82.02460887820014, "tpot_relative": 15.32199, "ttft_error_pct": 88.68, "ttft_ms": 358.6904028943331, "ttft_relative": 1.62553, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 76369.12621350825, "interactivity_tok_s": 12.531053740166072, "output_per_gpu": 764.6839202355417, "total_per_gpu": 6882.2108357047455, "tpot_error_pct": 10.02, "tpot_ms": 79.80174857878688, "tpot_relative": 14.906766, "ttft_error_pct": 14.14, "ttft_ms": 2720.928356139854, "ttft_relative": 12.330833, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 84999.99452062088, "interactivity_tok_s": 11.27566593452753, "output_per_gpu": 688.1535532821591, "total_per_gpu": 6193.431973263394, "tpot_ms": 88.68655792097141, "tpot_relative": 16.566426, "ttft_ms": 3168.8681484020008, "ttft_relative": 14.360828, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.2, "tpot_shape_error_pct": 15.8, "ttft_mape_pct": 17.69, "ttft_shape_error_pct": 29.18}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 54.1, "tpot_shape_error_pct": 2.91, "ttft_mape_pct": 4.98, "ttft_shape_error_pct": 6.43}, "framework": "vllm", "id": "d89a2d91f0e0cb0c", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 24456.28759002875, "interactivity_tok_s": 43.192855782877395, "output_per_gpu": 669.2757410439316, "total_per_gpu": null, "tpot_error_pct": 40.5, "tpot_ms": 23.15197691550699, "tpot_relative": 0.595041, "ttft_error_pct": 3.63, "ttft_ms": 771.8152054651006, "ttft_relative": 0.963725, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 56879.18769054301, "interactivity_tok_s": 16.43523955633987, "output_per_gpu": 256.14304995472304, "total_per_gpu": 2312.4804190620257, "tpot_error_pct": 56.38, "tpot_ms": 60.84486913452086, "tpot_relative": 1.563804, "ttft_error_pct": 1.82, "ttft_ms": 786.3084846890652, "ttft_relative": 0.981822, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 36650.49797874799, "interactivity_tok_s": 25.701500010629303, "output_per_gpu": 395.30192689705973, "total_per_gpu": 3568.8181495790113, "tpot_ms": 38.908234911831315, "tpot_relative": 1.0, "ttft_ms": 800.86649064433, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33801.262215464085, "interactivity_tok_s": 31.03533851429625, "output_per_gpu": 968.48454330866, "total_per_gpu": null, "tpot_error_pct": 49.9, "tpot_ms": 32.221333739902846, "tpot_relative": 0.828137, "ttft_error_pct": 31.75, "ttft_ms": 838.8377995434726, "ttft_relative": 1.047413, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 91400.31116271613, "interactivity_tok_s": 10.2418494324766, "output_per_gpu": 320.6445994099017, "total_per_gpu": 2884.971682602713, "tpot_error_pct": 51.83, "tpot_ms": 97.63861562239235, "tpot_relative": 2.509459, "ttft_error_pct": 8.13, "ttft_ms": 1129.082605249996, "ttft_relative": 1.409826, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 60661.643868528336, "interactivity_tok_s": 15.550069813254893, "output_per_gpu": 481.648212446373, "total_per_gpu": 4333.5875808956225, "tpot_ms": 64.30839295316855, "tpot_relative": 1.652822, "ttft_ms": 1229.0531764210755, "ttft_relative": 1.534654, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.17, "tpot_shape_error_pct": null, "ttft_mape_pct": 95.03, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.46, "tpot_shape_error_pct": null, "ttft_mape_pct": 74.84, "ttft_shape_error_pct": null}, "framework": "vllm", "id": "7572905ac5a81270", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 125104.80826458536, "interactivity_tok_s": 8.225898347882357, "output_per_gpu": 2093.3487979625097, "total_per_gpu": null, "tpot_error_pct": 19.17, "tpot_ms": 121.56726933751085, "tpot_relative": 0.80835, "ttft_error_pct": 95.03, "ttft_ms": 741.4917323117514, "ttft_relative": 0.049676, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 143282.235732138, "interactivity_tok_s": 6.618775917110965, "output_per_gpu": 1631.4376325317066, "total_per_gpu": 14683.0572152318, "tpot_error_pct": 0.46, "tpot_ms": 151.0853385162631, "tpot_relative": 1.004627, "ttft_error_pct": 74.84, "ttft_ms": 3754.8024964757687, "ttft_relative": 0.251553, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 153899.62559139804, "interactivity_tok_s": 6.649401296600651, "output_per_gpu": 1521.2791888571298, "total_per_gpu": 13691.623219250794, "tpot_ms": 150.3894795026473, "tpot_relative": 1.0, "ttft_ms": 14926.49880272773, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.55, "tpot_shape_error_pct": 2.21, "ttft_mape_pct": 38.16, "ttft_shape_error_pct": 3.95}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.0, "tpot_shape_error_pct": 4.35, "ttft_mape_pct": 4.69, "ttft_shape_error_pct": 2.29}, "framework": "vllm", "id": "2d0d3d25ed257089", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6847.123565410823, "interactivity_tok_s": 153.78910758650147, "output_per_gpu": 37.351450949703306, "total_per_gpu": null, "tpot_error_pct": 48.18, "tpot_ms": 6.502411098507297, "tpot_relative": 1.481818, "ttft_error_pct": 35.49, "ttft_ms": 195.15701163785798, "ttft_relative": 1.354887, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6179.877920800006, "interactivity_tok_s": 153.7519836053799, "output_per_gpu": 37.634238569213096, "total_per_gpu": 336.883677425539, "tpot_error_pct": 48.22, "tpot_ms": 6.503981129548232, "tpot_relative": 1.482176, "ttft_error_pct": 5.77, "ttft_ms": 135.73207130000063, "ttft_relative": 0.942326, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 4221.952526408131, "interactivity_tok_s": 227.88750107963656, "output_per_gpu": 55.083609771243815, "total_per_gpu": 493.0820904342793, "tpot_ms": 4.388130087268562, "tpot_relative": 1.0, "ttft_ms": 144.03932780551258, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7623.443799719322, "interactivity_tok_s": 138.2157680065467, "output_per_gpu": 67.09566089000778, "total_per_gpu": null, "tpot_error_pct": 44.91, "tpot_ms": 7.235064525724982, "tpot_relative": 1.648781, "ttft_error_pct": 40.84, "ttft_ms": 221.97278990266463, "ttft_relative": 1.541057, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6766.093266199992, "interactivity_tok_s": 141.27782869048542, "output_per_gpu": 68.91104078972133, "total_per_gpu": 614.098501908466, "tpot_error_pct": 41.77, "tpot_ms": 7.078251479861161, "tpot_relative": 1.613045, "ttft_error_pct": 3.61, "ttft_ms": 151.92311034999975, "ttft_relative": 1.054734, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 4823.476901500544, "interactivity_tok_s": 200.2931890853267, "output_per_gpu": 96.67951938485363, "total_per_gpu": 861.5563970458095, "tpot_ms": 4.992681002118305, "tpot_relative": 1.13777, "ttft_ms": 157.60841249866644, "ttft_relative": 1.094204, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.2, "tpot_shape_error_pct": 14.57, "ttft_mape_pct": 18.91, "ttft_shape_error_pct": 25.57}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.42, "tpot_shape_error_pct": 1.97, "ttft_mape_pct": 8.59, "ttft_shape_error_pct": 1.99}, "framework": "vllm", "id": "1ca8ed4b3809a005", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 26016.75555486347, "interactivity_tok_s": 40.22740165625733, "output_per_gpu": 629.1330202754751, "total_per_gpu": null, "tpot_error_pct": 12.85, "tpot_ms": 24.858677389730218, "tpot_relative": 0.871522, "ttft_error_pct": 7.02, "ttft_ms": 586.328585169455, "ttft_relative": 0.92976, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35240.12089496388, "interactivity_tok_s": 26.61660919825793, "output_per_gpu": 411.692078617358, "total_per_gpu": 3710.9009911014773, "tpot_error_pct": 31.72, "tpot_ms": 37.570525702629716, "tpot_relative": 1.317187, "ttft_error_pct": 7.67, "ttft_ms": 582.2644816062494, "ttft_relative": 0.923315, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 26935.9889398791, "interactivity_tok_s": 35.059052921958596, "output_per_gpu": 537.5379787863939, "total_per_gpu": 4845.247994404841, "tpot_ms": 28.52330330274462, "tpot_relative": 1.0, "ttft_ms": 630.6237567885546, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 33311.75274057344, "interactivity_tok_s": 31.30904654960884, "output_per_gpu": 982.7162279614251, "total_per_gpu": null, "tpot_error_pct": 25.55, "tpot_ms": 31.93965036321087, "tpot_relative": 1.119774, "ttft_error_pct": 30.8, "ttft_ms": 637.4904190087236, "ttft_relative": 1.010889, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51875.751861014214, "interactivity_tok_s": 18.052513166720885, "output_per_gpu": 560.161467250628, "total_per_gpu": 5057.1835728476935, "tpot_error_pct": 29.13, "tpot_ms": 55.393949350136005, "tpot_relative": 1.942059, "ttft_error_pct": 9.51, "ttft_ms": 833.6035831328121, "ttft_relative": 1.321872, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28196140241", "measured": {"e2e_ms": 40437.212830499084, "interactivity_tok_s": 23.310762396100376, "output_per_gpu": 717.5838548682311, "total_per_gpu": 6478.405772521071, "tpot_ms": 42.8986398217198, "tpot_relative": 1.503986, "ttft_ms": 921.1951316279738, "ttft_relative": 1.460768, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 61.92, "tpot_shape_error_pct": 3.83, "ttft_mape_pct": 32.53, "ttft_shape_error_pct": 7.51}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 60.53, "tpot_shape_error_pct": 1.73, "ttft_mape_pct": 10.58, "ttft_shape_error_pct": 6.13}, "framework": "vllm", "id": "4029c6fe2f70d7e8", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 6898.582585175731, "interactivity_tok_s": 152.28533900099092, "output_per_gpu": 18.536416491525205, "total_per_gpu": null, "tpot_error_pct": 58.88, "tpot_ms": 6.566620310005634, "tpot_relative": 1.588787, "ttft_error_pct": 27.73, "ttft_ms": 180.930008039967, "ttft_relative": 1.277298, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6235.846890399991, "interactivity_tok_s": 152.01962541815612, "output_per_gpu": 18.64822886832311, "total_per_gpu": 166.9300126022224, "tpot_error_pct": 59.16, "tpot_ms": 6.578098039969037, "tpot_relative": 1.591564, "ttft_error_pct": 13.24, "ttft_ms": 122.90145340000022, "ttft_relative": 0.867638, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 3982.6145379000536, "interactivity_tok_s": 241.94899962875112, "output_per_gpu": 29.19674867144952, "total_per_gpu": 261.35530929416643, "tpot_ms": 4.133102437019412, "tpot_relative": 1.0, "ttft_ms": 141.65053910001006, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7624.412955784632, "interactivity_tok_s": 137.8969011739355, "output_per_gpu": 33.543566105763304, "total_per_gpu": null, "tpot_error_pct": 64.96, "tpot_ms": 7.251794576142473, "tpot_relative": 1.754564, "ttft_error_pct": 37.32, "ttft_ms": 205.82710439088254, "ttft_relative": 1.453063, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6788.718612649995, "interactivity_tok_s": 140.49934823340533, "output_per_gpu": 34.34001497677895, "total_per_gpu": 306.01992817237675, "tpot_error_pct": 61.9, "tpot_ms": 7.117470739712931, "tpot_relative": 1.722065, "ttft_error_pct": 7.92, "ttft_ms": 138.01712775000016, "ttft_relative": 0.974349, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30841588542", "measured": {"e2e_ms": 4257.887930651486, "interactivity_tok_s": 227.4750718573239, "output_per_gpu": 54.74279762728764, "total_per_gpu": 487.83866312188985, "tpot_ms": 4.396086093456502, "tpot_relative": 1.063629, "ttft_ms": 149.88345655438025, "ttft_relative": 1.058121, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 101}]}, {"aic": {"coverage_pct": 92.5, "points": 37, "status_counts": {"failed": 3, "success": 37, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.44, "tpot_shape_error_pct": 3.33, "ttft_mape_pct": 64.44, "ttft_shape_error_pct": 22.17}, "aisimulate": {"coverage_pct": 100.0, "points": 40, "status_counts": {"failed": 0, "success": 40, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 55.65, "tpot_shape_error_pct": 4.87, "ttft_mape_pct": 60.13, "ttft_shape_error_pct": 34.33}, "frameworks": ["dynamo-trt", "sglang", "trt"], "gpu_skus": ["b200", "b300", "gb300"], "hf_model_paths": ["nvidia/Qwen3.5-397B-A17B-NVFP4"], "model": "nvidia/Qwen3.5-397B-A17B-NVFP4", "precisions": ["fp4"], "rows": 40, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.95, "tpot_shape_error_pct": 3.52, "ttft_mape_pct": 69.16, "ttft_shape_error_pct": 23.89}, "aisimulate": {"coverage_pct": 100.0, "points": 21, "status_counts": {"failed": 0, "success": 21, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 29.01, "tpot_shape_error_pct": 4.5, "ttft_mape_pct": 76.33, "ttft_shape_error_pct": 31.83}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.98, "tpot_shape_error_pct": 3.4, "ttft_mape_pct": 73.42, "ttft_shape_error_pct": 33.15}, "aisimulate": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 60.1, "tpot_shape_error_pct": 2.72, "ttft_mape_pct": 70.61, "ttft_shape_error_pct": 78.55}, "gpu": "b200", "precisions": ["fp4"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.51, "tpot_shape_error_pct": null, "ttft_mape_pct": 81.01, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 43.4, "tpot_shape_error_pct": null, "ttft_mape_pct": 43.32, "ttft_shape_error_pct": null}, "framework": "trt", "id": "f3728228f4cbab20", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 47447.900613421036, "interactivity_tok_s": 21.680497151348156, "output_per_gpu": 5537.915828318214, "total_per_gpu": null, "tpot_error_pct": 5.51, "tpot_ms": 46.124403560451434, "tpot_relative": 1.055142, "ttft_error_pct": 81.01, "ttft_ms": 262.63577107922197, "ttft_relative": 0.189901, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 58597.67360042855, "interactivity_tok_s": 15.952232368760349, "output_per_gpu": 3954.3886388995425, "total_per_gpu": 7908.544386405123, "tpot_error_pct": 43.4, "tpot_ms": 62.687151044660354, "tpot_relative": 1.434031, "ttft_error_pct": 43.32, "ttft_ms": 783.8785417017609, "ttft_relative": 0.56679, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 41678.23757201425, "interactivity_tok_s": 22.875997229471828, "output_per_gpu": 5468.444825684022, "total_per_gpu": 10936.56759052513, "tpot_ms": 43.71394129702334, "tpot_relative": 1.0, "ttft_ms": 1383.014349500027, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 84.56, "tpot_shape_error_pct": 2.43, "ttft_mape_pct": 73.3, "ttft_shape_error_pct": 15.78}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 86.32, "tpot_shape_error_pct": 2.04, "ttft_mape_pct": 82.85, "ttft_shape_error_pct": 19.76}, "framework": "trt", "id": "5b336b67d46cafa9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13185.229534197571, "interactivity_tok_s": 78.21810073365707, "output_per_gpu": 155.05718542818988, "total_per_gpu": null, "tpot_error_pct": 86.83, "tpot_ms": 12.784764531743512, "tpot_relative": 1.868344, "ttft_error_pct": 75.25, "ttft_ms": 106.41541822395769, "ttft_relative": 0.247502, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12025.297716787507, "interactivity_tok_s": 77.6303717605062, "output_per_gpu": 151.29363186765275, "total_per_gpu": 301.46975164134994, "tpot_error_pct": 88.25, "tpot_ms": 12.881556243026285, "tpot_relative": 1.882489, "ttft_error_pct": 84.39, "ttft_ms": 67.11551677500054, "ttft_relative": 0.156098, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 6783.954148725752, "interactivity_tok_s": 146.1383573582697, "output_per_gpu": 267.5056791903418, "total_per_gpu": 533.0354600695207, "tpot_ms": 6.842830438749364, "tpot_relative": 1.0, "ttft_ms": 429.9583063722821, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15586.204215632528, "interactivity_tok_s": 66.16867161642784, "output_per_gpu": 262.76952040327933, "total_per_gpu": null, "tpot_error_pct": 82.29, "tpot_ms": 15.11289218252536, "tpot_relative": 2.208573, "ttft_error_pct": 71.34, "ttft_ms": 125.71551290908518, "ttft_relative": 0.29239, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14072.195547362417, "interactivity_tok_s": 65.4131173842894, "output_per_gpu": 254.77804629049444, "total_per_gpu": 512.2682914046322, "tpot_error_pct": 84.4, "tpot_ms": 15.287453648252134, "tpot_relative": 2.234083, "ttft_error_pct": 81.31, "ttft_ms": 82.01136931249926, "ttft_relative": 0.190743, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 8029.650201788536, "interactivity_tok_s": 120.62210118209248, "output_per_gpu": 446.5147395540457, "total_per_gpu": 897.7827801439943, "tpot_ms": 8.290354671325023, "tpot_relative": 1.211539, "ttft_ms": 438.6986423352937, "ttft_relative": 1.020328, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.02, "tpot_shape_error_pct": 4.78, "ttft_mape_pct": 67.88, "ttft_shape_error_pct": 2.72}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 36.52, "tpot_shape_error_pct": 2.17, "ttft_mape_pct": 73.24, "ttft_shape_error_pct": 40.24}, "framework": "trt", "id": "8666d7d6a9b1f2a6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 16827.55769056426, "interactivity_tok_s": 61.31849845167261, "output_per_gpu": 976.6752284246218, "total_per_gpu": null, "tpot_error_pct": 33.2, "tpot_ms": 16.30829236283627, "tpot_relative": 1.332035, "ttft_error_pct": 67.44, "ttft_ms": 144.17460338275515, "ttft_relative": 0.325636, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15667.1954880827, "interactivity_tok_s": 59.1781964885796, "output_per_gpu": 916.0307681650572, "total_per_gpu": 1832.4745537742028, "tpot_error_pct": 38.02, "tpot_ms": 16.898115511056226, "tpot_relative": 1.38021, "ttft_error_pct": 77.72, "ttft_ms": 98.64684822968759, "ttft_relative": 0.222806, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 11719.323820427599, "interactivity_tok_s": 81.67835784843125, "output_per_gpu": 1220.7051026039867, "total_per_gpu": 2441.960593381586, "tpot_ms": 12.243145263224786, "tpot_relative": 1.0, "ttft_ms": 442.7471009533292, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20617.858084688236, "interactivity_tok_s": 49.97740684351031, "output_per_gpu": 1593.7148896787112, "total_per_gpu": null, "tpot_error_pct": 26.83, "tpot_ms": 20.009041348047703, "tpot_relative": 1.634306, "ttft_error_pct": 68.32, "ttft_ms": 148.60878563543412, "ttft_relative": 0.335652, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19734.246012594485, "interactivity_tok_s": 46.94717373848597, "output_per_gpu": 1453.398048571955, "total_per_gpu": 2909.98897890238, "tpot_error_pct": 35.02, "tpot_ms": 21.300536760112315, "tpot_relative": 1.739793, "ttft_error_pct": 68.75, "ttft_ms": 146.58975374453095, "ttft_relative": 0.331091, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 14971.585351861677, "interactivity_tok_s": 63.38770170533187, "output_per_gpu": 1905.795583228735, "total_per_gpu": 3815.7778928391667, "tpot_ms": 15.775930868241035, "tpot_relative": 1.288552, "ttft_ms": 469.13041841276026, "ttft_relative": 1.05959, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 30.49, "tpot_shape_error_pct": 4.69, "ttft_mape_pct": 77.21, "ttft_shape_error_pct": 97.98}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.25, "tpot_shape_error_pct": 1.86, "ttft_mape_pct": 85.28, "ttft_shape_error_pct": 114.66}, "framework": "trt", "id": "904b23d0e526070b", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7380.660706894921, "interactivity_tok_s": 139.73876605671967, "output_per_gpu": 69.1768963001966, "total_per_gpu": null, "tpot_error_pct": 33.62, "tpot_ms": 7.1562103217234805, "tpot_relative": 1.336218, "ttft_error_pct": 84.7, "ttft_ms": 59.857547771800185, "ttft_relative": 0.152992, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6640.499137975012, "interactivity_tok_s": 138.8223776862889, "output_per_gpu": 66.89461878704704, "total_per_gpu": 134.44344139725868, "tpot_error_pct": 34.5, "tpot_ms": 7.203449592686002, "tpot_relative": 1.345038, "ttft_error_pct": 90.64, "ttft_ms": 36.604052200000524, "ttft_relative": 0.093558, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 5302.64424497509, "interactivity_tok_s": 186.7213943280289, "output_per_gpu": 83.82572724905211, "total_per_gpu": 168.47123809565275, "tpot_ms": 5.3555726894542, "tpot_relative": 1.0, "ttft_ms": 391.24625302647473, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13363.593392084798, "interactivity_tok_s": 77.2414521256861, "output_per_gpu": 614.9754549912924, "total_per_gpu": null, "tpot_error_pct": 27.35, "tpot_ms": 12.946416366859797, "tpot_relative": 2.417373, "ttft_error_pct": 69.71, "ttft_ms": 119.40944878722686, "ttft_relative": 0.305203, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12442.823774953138, "interactivity_tok_s": 74.52087506339973, "output_per_gpu": 575.7589502031996, "total_per_gpu": 1151.777497025081, "tpot_error_pct": 32.0, "tpot_ms": 13.41905874225491, "tpot_relative": 2.505625, "ttft_error_pct": 79.92, "ttft_ms": 79.17322497499988, "ttft_relative": 0.202362, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 9758.192183361916, "interactivity_tok_s": 98.3687993029302, "output_per_gpu": 731.877245075277, "total_per_gpu": 1464.0844769584785, "tpot_ms": 10.165825008399915, "tpot_relative": 1.898177, "ttft_ms": 394.23737491551947, "ttft_relative": 1.007645, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.59, "tpot_shape_error_pct": 1.71, "ttft_mape_pct": 71.52, "ttft_shape_error_pct": 16.13}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 92.67, "tpot_shape_error_pct": 4.79, "ttft_mape_pct": 54.7, "ttft_shape_error_pct": 139.55}, "framework": "trt", "id": "a21f96a7019361be", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 23836.119187328528, "interactivity_tok_s": 43.400092312451726, "output_per_gpu": 2759.75429081252, "total_per_gpu": null, "tpot_error_pct": 19.61, "tpot_ms": 23.041425644919528, "tpot_relative": 1.196066, "ttft_error_pct": 69.03, "ttft_ms": 264.74075257584866, "ttft_relative": 0.309749, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35324.41630434215, "interactivity_tok_s": 26.29725533507982, "output_per_gpu": 1636.1560253481855, "total_per_gpu": 3270.8048580434697, "tpot_error_pct": 97.39, "tpot_ms": 38.026782158745945, "tpot_relative": 1.973946, "ttft_error_pct": 73.32, "ttft_ms": 228.06167174121214, "ttft_relative": 0.266834, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 18617.394625773253, "interactivity_tok_s": 51.90936055573822, "output_per_gpu": 2963.9496048237233, "total_per_gpu": 5925.168881366555, "tpot_ms": 19.264348265786083, "tpot_relative": 1.0, "ttft_ms": 854.6951423828773, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32416.144977546657, "interactivity_tok_s": 31.829201613631707, "output_per_gpu": 4055.9592962387933, "total_per_gpu": null, "tpot_error_pct": 17.57, "tpot_ms": 31.417690337910432, "tpot_relative": 1.630872, "ttft_error_pct": 74.02, "ttft_ms": 275.8477618642863, "ttft_relative": 0.322744, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 47016.63696398979, "interactivity_tok_s": 19.910916245222758, "output_per_gpu": 2463.7014661784146, "total_per_gpu": 4927.257834107477, "tpot_error_pct": 87.94, "tpot_ms": 50.223705814639786, "tpot_relative": 2.60708, "ttft_error_pct": 36.08, "ttft_ms": 678.7057301376024, "ttft_relative": 0.794091, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 25701.68366267501, "interactivity_tok_s": 37.420233297826286, "output_per_gpu": 4434.72073858043, "total_per_gpu": 8869.180296890292, "tpot_ms": 26.723510568227518, "tpot_relative": 1.3872, "ttft_ms": 1061.8200103955644, "ttft_relative": 1.242338, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.68, "tpot_shape_error_pct": 3.57, "ttft_mape_pct": 65.97, "ttft_shape_error_pct": 20.19}, "aisimulate": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.69, "tpot_shape_error_pct": 5.21, "ttft_mape_pct": 80.63, "ttft_shape_error_pct": 13.14}, "gpu": "b300", "precisions": ["fp4"], "rows": 12, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.25, "tpot_shape_error_pct": 3.6, "ttft_mape_pct": 65.76, "ttft_shape_error_pct": 23.54}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.69, "tpot_shape_error_pct": 5.57, "ttft_mape_pct": 80.28, "ttft_shape_error_pct": 13.21}, "framework": "sglang", "id": "c18c335e9bffe588", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 7010.9465455654645, "interactivity_tok_s": 147.47016716079892, "output_per_gpu": 290.9699432584366, "total_per_gpu": null, "tpot_error_pct": 1.79, "tpot_ms": 6.781032525104669, "tpot_relative": 1.017884, "ttft_error_pct": 58.78, "ttft_ms": 73.95027238338834, "ttft_relative": 0.412182, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6279.638179924995, "interactivity_tok_s": 146.88526499925786, "output_per_gpu": 284.0046209912598, "total_per_gpu": 570.7866987080055, "tpot_error_pct": 2.19, "tpot_ms": 6.808034829123619, "tpot_relative": 1.021938, "ttft_error_pct": 78.74, "ttft_ms": 38.14301662499978, "ttft_relative": 0.212601, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 6287.66592919128, "interactivity_tok_s": 150.1075668675554, "output_per_gpu": 283.3391120233437, "total_per_gpu": 569.4491723486401, "tpot_ms": 6.661889342875908, "tpot_relative": 1.0, "ttft_ms": 179.41149852704257, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8701.553453379809, "interactivity_tok_s": 118.67406335025215, "output_per_gpu": 469.70764551290955, "total_per_gpu": null, "tpot_error_pct": 4.07, "tpot_ms": 8.426441058553973, "tpot_relative": 1.264873, "ttft_error_pct": 55.47, "ttft_ms": 81.30425047909515, "ttft_relative": 0.453172, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7909.146950400003, "interactivity_tok_s": 118.02983309354877, "output_per_gpu": 461.23650775560293, "total_per_gpu": 919.0661478907524, "tpot_error_pct": 4.63, "tpot_ms": 8.472434246411364, "tpot_relative": 1.271776, "ttft_error_pct": 76.35, "ttft_ms": 43.19014553750009, "ttft_relative": 0.240732, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 7702.006703073857, "interactivity_tok_s": 123.49973801873989, "output_per_gpu": 472.86342400914856, "total_per_gpu": 942.2340995886625, "tpot_ms": 8.097183168504047, "tpot_relative": 1.215448, "ttft_ms": 182.59373290347867, "ttft_relative": 1.017737, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11146.292768719884, "interactivity_tok_s": 92.57727278982921, "output_per_gpu": 734.8229330697252, "total_per_gpu": null, "tpot_error_pct": 2.13, "tpot_ms": 10.801787197493063, "tpot_relative": 1.62143, "ttft_error_pct": 68.68, "ttft_ms": 96.06446568448084, "ttft_relative": 0.535442, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10029.854880256291, "interactivity_tok_s": 91.70228473358357, "output_per_gpu": 716.4553440901049, "total_per_gpu": 1440.537598621381, "tpot_error_pct": 3.1, "tpot_ms": 10.904853711172326, "tpot_relative": 1.636901, "ttft_error_pct": 83.69, "ttft_ms": 50.02301634375063, "ttft_relative": 0.278817, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 9990.736093443502, "interactivity_tok_s": 94.5449377461049, "output_per_gpu": 719.3959298903848, "total_per_gpu": 1446.4500737563876, "tpot_ms": 10.576980892255104, "tpot_relative": 1.587685, "ttft_ms": 306.74278564360975, "ttft_relative": 1.709716, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13900.655606387429, "interactivity_tok_s": 74.26662503919337, "output_per_gpu": 1181.1828416380124, "total_per_gpu": null, "tpot_error_pct": 0.52, "tpot_ms": 13.464998570653524, "tpot_relative": 2.021198, "ttft_error_pct": 65.61, "ttft_ms": 125.96206860887241, "ttft_relative": 0.702085, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12734.28263998758, "interactivity_tok_s": 72.83631098463277, "output_per_gpu": 1135.7314844662885, "total_per_gpu": 2267.870353243558, "tpot_error_pct": 1.44, "tpot_ms": 13.729415815841952, "tpot_relative": 2.060889, "ttft_error_pct": 82.7, "ttft_ms": 63.35852724375002, "ttft_relative": 0.353146, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 12860.753242834835, "interactivity_tok_s": 73.8840576485097, "output_per_gpu": 1125.0681065264928, "total_per_gpu": 2246.5773284169563, "tpot_ms": 13.534719556921504, "tpot_relative": 2.031664, "ttft_ms": 366.27362702238315, "ttft_relative": 2.041528, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19809.34680779384, "interactivity_tok_s": 52.03284867862563, "output_per_gpu": 1658.8849725656207, "total_per_gpu": null, "tpot_error_pct": 10.23, "tpot_ms": 19.218628720029816, "tpot_relative": 2.884862, "ttft_error_pct": 68.76, "ttft_ms": 148.68962720334002, "ttft_relative": 0.828763, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18448.964297920396, "interactivity_tok_s": 50.17885600876348, "output_per_gpu": 1565.4438713942889, "total_per_gpu": 3131.593565834153, "tpot_error_pct": 14.3, "tpot_ms": 19.928712600091064, "tpot_relative": 2.991451, "ttft_error_pct": 81.1, "ttft_ms": 89.93051088906263, "ttft_relative": 0.501253, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 16536.82666924069, "interactivity_tok_s": 57.35476017462461, "output_per_gpu": 1737.266138841459, "total_per_gpu": 3475.315571481884, "tpot_ms": 17.435344458861998, "tpot_relative": 2.617177, "ttft_ms": 475.9461443085456, "ttft_relative": 2.652819, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25410.112771377957, "interactivity_tok_s": 40.50524422445428, "output_per_gpu": 2585.5567764577863, "total_per_gpu": null, "tpot_error_pct": 6.75, "tpot_ms": 24.68816122817669, "tpot_relative": 3.70588, "ttft_error_pct": 77.28, "ttft_ms": 154.12383495320245, "ttft_relative": 0.859052, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24480.346834275846, "interactivity_tok_s": 37.776605370524514, "output_per_gpu": 2353.133521162922, "total_per_gpu": 4711.436498210519, "tpot_error_pct": 14.47, "tpot_ms": 26.471409757221267, "tpot_relative": 3.973559, "ttft_error_pct": 79.11, "ttft_ms": 141.6873140999998, "ttft_relative": 0.789734, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 21940.82684002551, "interactivity_tok_s": 43.24118741686278, "output_per_gpu": 2616.279498591916, "total_per_gpu": 5238.306542458384, "tpot_ms": 23.126099437547584, "tpot_relative": 3.471403, "ttft_ms": 678.2561419668127, "ttft_relative": 3.78045, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.11, "tpot_shape_error_pct": 3.54, "ttft_mape_pct": 66.17, "ttft_shape_error_pct": 16.84}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.69, "tpot_shape_error_pct": 4.85, "ttft_mape_pct": 80.97, "ttft_shape_error_pct": 13.07}, "framework": "sglang", "id": "58922e229a16f4ec", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5598.801216887574, "interactivity_tok_s": 184.78436520029888, "output_per_gpu": 182.1258149576834, "total_per_gpu": null, "tpot_error_pct": 0.45, "tpot_ms": 5.411713263273329, "tpot_relative": 1.004549, "ttft_error_pct": 62.08, "ttft_ms": 62.61854855895808, "ttft_relative": 0.379199, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5025.903023775, "interactivity_tok_s": 183.5970060808355, "output_per_gpu": 176.9945925812835, "total_per_gpu": 355.72012467976117, "tpot_error_pct": 1.1, "tpot_ms": 5.446711911847365, "tpot_relative": 1.011046, "ttft_error_pct": 80.36, "ttft_ms": 32.43566849999992, "ttft_relative": 0.196421, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 5104.930190113373, "interactivity_tok_s": 185.62501234239156, "output_per_gpu": 174.42666490770074, "total_per_gpu": 350.5591559806979, "tpot_ms": 5.387205029004747, "tpot_relative": 1.0, "ttft_ms": 165.1337789138779, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6787.176617895191, "interactivity_tok_s": 152.2700801628861, "output_per_gpu": 301.01549246414396, "total_per_gpu": null, "tpot_error_pct": 3.19, "tpot_ms": 6.567278344703579, "tpot_relative": 1.219051, "ttft_error_pct": 58.38, "ttft_ms": 68.85087126342972, "ttft_relative": 0.41694, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6173.669475312505, "interactivity_tok_s": 151.2658870723818, "output_per_gpu": 295.0210460174889, "total_per_gpu": 587.8629547981616, "tpot_error_pct": 3.87, "tpot_ms": 6.610875851483243, "tpot_relative": 1.227144, "ttft_error_pct": 78.12, "ttft_ms": 36.194661512500105, "ttft_relative": 0.219184, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 6076.171940402128, "interactivity_tok_s": 157.12424323428885, "output_per_gpu": 299.5622706921774, "total_per_gpu": 596.9118609413065, "tpot_ms": 6.364390239314593, "tpot_relative": 1.18139, "ttft_ms": 165.44376562233083, "ttft_relative": 1.001877, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8230.073319076293, "interactivity_tok_s": 125.54303972084267, "output_per_gpu": 497.51345591090404, "total_per_gpu": null, "tpot_error_pct": 0.49, "tpot_ms": 7.965395789552321, "tpot_relative": 1.478577, "ttft_error_pct": 68.37, "ttft_ms": 81.47342636426848, "ttft_relative": 0.493378, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7417.976245693743, "interactivity_tok_s": 124.06544724238434, "output_per_gpu": 483.7729271570923, "total_per_gpu": 972.6957814097474, "tpot_error_pct": 0.69, "tpot_ms": 8.060261920035792, "tpot_relative": 1.496186, "ttft_error_pct": 83.92, "ttft_ms": 41.41099542500025, "ttft_relative": 0.250772, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 7586.784688904299, "interactivity_tok_s": 124.92418297865999, "output_per_gpu": 473.19694021195187, "total_per_gpu": 951.4312225469029, "tpot_ms": 8.004855234240946, "tpot_relative": 1.485901, "ttft_ms": 257.56981729355175, "ttft_relative": 1.559765, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10154.22585291988, "interactivity_tok_s": 101.81788251305237, "output_per_gpu": 808.6442940635923, "total_per_gpu": null, "tpot_error_pct": 4.56, "tpot_ms": 9.821457442623664, "tpot_relative": 1.823108, "ttft_error_pct": 66.65, "ttft_ms": 106.87488911587195, "ttft_relative": 0.647202, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9297.968616631206, "interactivity_tok_s": 99.81974110363998, "output_per_gpu": 776.7033840226289, "total_per_gpu": 1550.94985212691, "tpot_error_pct": 2.65, "tpot_ms": 10.0180584415835, "tpot_relative": 1.859602, "ttft_error_pct": 83.89, "ttft_ms": 51.62301054687494, "ttft_relative": 0.312613, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 9821.395588156884, "interactivity_tok_s": 97.17209575669555, "output_per_gpu": 736.5590761944876, "total_per_gpu": 1470.788223414373, "tpot_ms": 10.291020196825341, "tpot_relative": 1.910271, "ttft_ms": 320.4598238138715, "ttft_relative": 1.940607, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14119.004060725065, "interactivity_tok_s": 73.11160181031578, "output_per_gpu": 1164.155744320918, "total_per_gpu": null, "tpot_error_pct": 4.04, "tpot_ms": 13.677719749520024, "tpot_relative": 2.538927, "ttft_error_pct": 67.49, "ttft_ms": 126.69675696607835, "ttft_relative": 0.767237, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13088.016602182839, "interactivity_tok_s": 70.78476311645093, "output_per_gpu": 1101.138123695872, "total_per_gpu": 2202.7727255333557, "tpot_error_pct": 7.46, "tpot_ms": 14.127334131991919, "tpot_relative": 2.622387, "ttft_error_pct": 81.53, "ttft_ms": 71.98731286093722, "ttft_relative": 0.435933, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 12501.79870934735, "interactivity_tok_s": 76.06342801927532, "output_per_gpu": 1148.5896585491544, "total_per_gpu": 2297.6971900580024, "tpot_ms": 13.146922588692542, "tpot_relative": 2.440398, "ttft_ms": 389.69345494697336, "ttft_relative": 2.359865, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18726.830868575194, "interactivity_tok_s": 55.01526409608392, "output_per_gpu": 1754.5778401425746, "total_per_gpu": null, "tpot_error_pct": 5.94, "tpot_ms": 18.176773599659622, "tpot_relative": 3.374064, "ttft_error_pct": 74.05, "ttft_ms": 131.99147612339902, "ttft_relative": 0.7993, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17834.354191641458, "interactivity_tok_s": 51.8849570616689, "output_per_gpu": 1614.5373248467147, "total_per_gpu": 3232.6215285253757, "tpot_error_pct": 12.33, "tpot_ms": 19.273409030895603, "tpot_relative": 3.577627, "ttft_error_pct": 78.03, "ttft_ms": 111.7682739937499, "ttft_relative": 0.676835, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719981652", "measured": {"e2e_ms": 16283.808348120874, "interactivity_tok_s": 58.283882018441076, "output_per_gpu": 1763.0517736412994, "total_per_gpu": 3529.976688472535, "tpot_ms": 17.157402104472023, "tpot_relative": 3.184843, "ttft_ms": 508.68740680816705, "ttft_relative": 3.080456, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 21}, {"aic": {"coverage_pct": 84.21, "points": 16, "status_counts": {"failed": 3, "success": 16, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 60.7, "tpot_shape_error_pct": 2.65, "ttft_mape_pct": 58.25, "ttft_shape_error_pct": 16.12}, "aisimulate": {"coverage_pct": 100.0, "points": 19, "status_counts": {"failed": 0, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 85.09, "tpot_shape_error_pct": 6.16, "ttft_mape_pct": 42.21, "ttft_shape_error_pct": 43.1}, "gpu_skus": ["b200", "gb300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.88, "tpot_shape_error_pct": 2.65, "ttft_mape_pct": 39.94, "ttft_shape_error_pct": 16.12}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 108.04, "tpot_shape_error_pct": 6.16, "ttft_mape_pct": 50.13, "ttft_shape_error_pct": 43.1}, "gpu": "b200", "precisions": ["fp4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.83, "tpot_shape_error_pct": null, "ttft_mape_pct": 83.02, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 66.26, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.29, "ttft_shape_error_pct": null}, "framework": "trt", "id": "95a1dec0f686fb44", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 109346.79929483024, "interactivity_tok_s": 9.457175519002451, "output_per_gpu": 2406.139320019889, "total_per_gpu": null, "tpot_error_pct": 2.83, "tpot_ms": 105.7398160783507, "tpot_relative": 1.028341, "ttft_error_pct": 83.02, "ttft_ms": 1174.96744667747, "ttft_relative": 0.169786, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 162557.2916241964, "interactivity_tok_s": 5.849337389238094, "output_per_gpu": 1441.665005263978, "total_per_gpu": 12972.32407467688, "tpot_error_pct": 66.26, "tpot_ms": 170.95953497909872, "tpot_relative": 1.662616, "ttft_error_pct": 33.29, "ttft_ms": 4616.33029192438, "ttft_relative": 0.667072, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 101614.28315888811, "interactivity_tok_s": 9.725201990822757, "output_per_gpu": 2286.1255790554033, "total_per_gpu": 20570.910564264384, "tpot_ms": 102.82562778065234, "tpot_relative": 1.0, "ttft_ms": 6920.29263017825, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 89.1, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.4, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 94.09, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.98, "ttft_shape_error_pct": null}, "framework": "trt", "id": "c9cc55dd57b891b1", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 11960.130861168896, "interactivity_tok_s": 88.3894535330393, "output_per_gpu": 84.43000983750537, "total_per_gpu": null, "tpot_error_pct": 89.1, "tpot_ms": 11.313566947511534, "tpot_relative": 1.890969, "ttft_error_pct": 16.4, "ttft_ms": 386.35187386459864, "ttft_relative": 0.835955, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10885.655706774985, "interactivity_tok_s": 86.1154895186244, "output_per_gpu": 82.45244027923303, "total_per_gpu": 741.1982645995284, "tpot_error_pct": 94.09, "tpot_ms": 11.612312785886536, "tpot_relative": 1.940902, "ttft_error_pct": 53.98, "ttft_ms": 212.6752606250001, "ttft_relative": 0.460169, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 5962.516755772231, "interactivity_tok_s": 167.14171833345318, "output_per_gpu": 151.06295278321483, "total_per_gpu": 1357.9658536364773, "tpot_ms": 5.982946747053105, "tpot_relative": 1.0, "ttft_ms": 462.16799849498784, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 71.2, "tpot_shape_error_pct": 2.04, "ttft_mape_pct": 19.09, "ttft_shape_error_pct": 0.16}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.73, "tpot_shape_error_pct": 3.38, "ttft_mape_pct": 53.29, "ttft_shape_error_pct": 10.34}, "framework": "trt", "id": "5653aa94606aa829", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 13446.36104559058, "interactivity_tok_s": 78.79868129446365, "output_per_gpu": 150.04441133486222, "total_per_gpu": null, "tpot_error_pct": 72.96, "tpot_ms": 12.690567704592532, "tpot_relative": 1.72965, "ttft_error_pct": 19.15, "ttft_ms": 463.91028379242005, "ttft_relative": 0.808456, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12240.516958600068, "interactivity_tok_s": 76.68354362856178, "output_per_gpu": 146.7703124498503, "total_per_gpu": 1319.3775771116757, "tpot_error_pct": 77.74, "tpot_ms": 13.040607576037175, "tpot_relative": 1.777358, "ttft_error_pct": 55.59, "ttft_ms": 254.8484557750001, "ttft_relative": 0.444124, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 7317.35748557403, "interactivity_tok_s": 136.29411888799316, "output_per_gpu": 246.5088923474, "total_per_gpu": 2215.967927661976, "tpot_ms": 7.337073735527814, "tpot_relative": 1.0, "ttft_ms": 573.8227869020193, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21255.76234544019, "interactivity_tok_s": 49.52758113564216, "output_per_gpu": 384.3208739974764, "total_per_gpu": null, "tpot_error_pct": 69.43, "tpot_ms": 20.19077001279914, "tpot_relative": 2.751883, "ttft_error_pct": 19.03, "ttft_ms": 600.6046223466714, "ttft_relative": 1.046673, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19087.41797787496, "interactivity_tok_s": 48.864459258173355, "output_per_gpu": 372.76355693762724, "total_per_gpu": 3364.0977997328546, "tpot_error_pct": 71.73, "tpot_ms": 20.464771639373748, "tpot_relative": 2.789228, "ttft_error_pct": 50.99, "ttft_ms": 363.50167875624976, "ttft_relative": 0.633474, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 11649.98816770676, "interactivity_tok_s": 83.91493144926127, "output_per_gpu": 606.2191503733493, "total_per_gpu": 5470.976097237277, "tpot_ms": 11.916830327206366, "tpot_relative": 1.624194, "ttft_ms": 741.738339108997, "ttft_relative": 1.292626, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.24, "tpot_shape_error_pct": null, "ttft_mape_pct": 44.06, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.69, "tpot_shape_error_pct": null, "ttft_mape_pct": 69.52, "ttft_shape_error_pct": null}, "framework": "trt", "id": "1a7d65574d03722d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7690.559291199922, "interactivity_tok_s": 137.3864557440824, "output_per_gpu": 65.66729821173959, "total_per_gpu": null, "tpot_error_pct": 31.24, "tpot_ms": 7.278737882741201, "tpot_relative": 1.312417, "ttft_error_pct": 44.06, "ttft_ms": 244.41043715567412, "ttft_relative": 0.559429, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6998.838696349999, "interactivity_tok_s": 133.87054751783742, "output_per_gpu": 64.09704270852204, "total_per_gpu": 576.1941873475903, "tpot_error_pct": 34.69, "tpot_ms": 7.469902966272369, "tpot_relative": 1.346885, "ttft_error_pct": 69.52, "ttft_ms": 133.1727637250005, "ttft_relative": 0.304818, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 5534.9044455826515, "interactivity_tok_s": 180.30827553727121, "output_per_gpu": 81.5623841696512, "total_per_gpu": 733.1971909917115, "tpot_ms": 5.5460571458534735, "tpot_relative": 1.0, "ttft_ms": 436.8923836518661, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 41.9, "tpot_shape_error_pct": 6.42, "ttft_mape_pct": 24.94, "ttft_shape_error_pct": 12.62}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.44, "tpot_shape_error_pct": 6.96, "ttft_mape_pct": 54.04, "ttft_shape_error_pct": 24.96}, "framework": "trt", "id": "f91251f65c3491c0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 13964.716388975787, "interactivity_tok_s": 75.5892692217326, "output_per_gpu": 290.5849469593873, "total_per_gpu": null, "tpot_error_pct": 46.6, "tpot_ms": 13.229391027271515, "tpot_relative": 1.465991, "ttft_error_pct": 29.4, "ttft_ms": 431.04936807702734, "ttft_relative": 0.706031, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12751.981620449973, "interactivity_tok_s": 74.04873368052453, "output_per_gpu": 285.86837172356405, "total_per_gpu": 2542.9313346268805, "tpot_error_pct": 49.65, "tpot_ms": 13.504619867159306, "tpot_relative": 1.49649, "ttft_error_pct": 59.14, "ttft_ms": 249.47116242500016, "ttft_relative": 0.408618, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 8964.418106238008, "interactivity_tok_s": 110.81321239296285, "output_per_gpu": 407.3658230187158, "total_per_gpu": 3623.7073369280724, "tpot_ms": 9.024194664205083, "tpot_relative": 1.0, "ttft_ms": 610.5244070626213, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23500.88967701131, "interactivity_tok_s": 44.79460088530889, "output_per_gpu": 700.3934417145322, "total_per_gpu": null, "tpot_error_pct": 37.19, "tpot_ms": 22.324118983901162, "tpot_relative": 2.473807, "ttft_error_pct": 20.49, "ttft_ms": 663.3159564804245, "ttft_relative": 1.086469, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21395.21348461893, "interactivity_tok_s": 44.136911003049164, "output_per_gpu": 675.7804192443086, "total_per_gpu": 6043.043682088972, "tpot_error_pct": 39.24, "tpot_ms": 22.656773600012826, "tpot_relative": 2.51067, "ttft_error_pct": 48.94, "ttft_ms": 425.9689311125001, "ttft_relative": 0.69771, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 15897.179870489526, "interactivity_tok_s": 61.45584545850952, "output_per_gpu": 904.129137489215, "total_per_gpu": 8085.010628462016, "tpot_ms": 16.27184513595418, "tpot_relative": 1.803135, "ttft_ms": 834.2485467454026, "ttft_relative": 1.366446, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.49, "tpot_shape_error_pct": 1.07, "ttft_mape_pct": 55.95, "ttft_shape_error_pct": 25.85}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 215.67, "tpot_shape_error_pct": 7.16, "ttft_mape_pct": 43.27, "ttft_shape_error_pct": 68.54}, "framework": "trt", "id": "34d81bc87b3110d0", "is_multinode": false, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 29126.437298121604, "interactivity_tok_s": 36.62422949744763, "output_per_gpu": 1132.1038445360625, "total_per_gpu": null, "tpot_error_pct": 15.67, "tpot_ms": 27.304328684093974, "tpot_relative": 1.156654, "ttft_error_pct": 46.78, "ttft_ms": 1194.1090542934699, "ttft_relative": 0.532234, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 72520.62319636447, "interactivity_tok_s": 12.901991315259828, "output_per_gpu": 403.50252114745115, "total_per_gpu": 3630.478571950185, "tpot_error_pct": 228.33, "tpot_ms": 77.50741537217206, "tpot_relative": 3.283337, "ttft_error_pct": 61.06, "ttft_ms": 873.545231810547, "ttft_relative": 0.389353, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 24026.93732154726, "interactivity_tok_s": 42.3615797427342, "output_per_gpu": 1169.3498469326273, "total_per_gpu": 10521.1227685261, "tpot_ms": 23.606296225803963, "tpot_relative": 1.0, "ttft_ms": 2243.5807363432104, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 43193.68485461759, "interactivity_tok_s": 24.4730714472013, "output_per_gpu": 1531.0144489212057, "total_per_gpu": null, "tpot_error_pct": 16.43, "tpot_ms": 40.861238122784066, "tpot_relative": 1.730947, "ttft_error_pct": 52.23, "ttft_ms": 1392.6382550094913, "ttft_relative": 0.620721, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 109368.15055643147, "interactivity_tok_s": 8.561179765398212, "output_per_gpu": 535.9783784016531, "total_per_gpu": 4823.844343952072, "tpot_error_pct": 232.84, "tpot_ms": 116.80633130047192, "tpot_relative": 4.948101, "ttft_error_pct": 52.05, "ttft_ms": 1397.8799879205108, "ttft_relative": 0.623058, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 35304.91533751846, "interactivity_tok_s": 28.49512859034003, "output_per_gpu": 1576.9200198064586, "total_per_gpu": 14192.394740050047, "tpot_ms": 35.09371775002287, "tpot_relative": 1.486625, "ttft_ms": 2915.429193386558, "ttft_relative": 1.299454, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 70821.26170366303, "interactivity_tok_s": 14.737684712818675, "output_per_gpu": 1861.7530145112012, "total_per_gpu": null, "tpot_error_pct": 17.36, "tpot_ms": 67.85326321509721, "tpot_relative": 2.874371, "ttft_error_pct": 68.84, "ttft_ms": 1407.373434618575, "ttft_relative": 0.627289, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 156492.9605837409, "interactivity_tok_s": 6.051001164465664, "output_per_gpu": 749.6473954242556, "total_per_gpu": 6745.442886990409, "tpot_error_pct": 185.85, "tpot_ms": 165.26190837186948, "tpot_relative": 7.000756, "ttft_error_pct": 16.7, "ttft_ms": 3762.4039757772553, "ttft_relative": 1.676964, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27582708402", "measured": {"e2e_ms": 57822.183859187455, "interactivity_tok_s": 17.296556860203747, "output_per_gpu": 1994.3512638959419, "total_per_gpu": 17945.480274221114, "tpot_ms": 57.814974857846956, "tpot_relative": 2.449134, "ttft_ms": 4516.75263256372, "ttft_relative": 2.013189, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 66.67, "points": 6, "status_counts": {"failed": 3, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 95.39, "tpot_shape_error_pct": null, "ttft_mape_pct": 88.77, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 59.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.42, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp4"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 119.49, "tpot_shape_error_pct": null, "ttft_mape_pct": 86.51, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 94.44, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.24, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "4945c2817b5e9c10", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 17202.284, "interactivity_tok_s": 61.24448799608035, "output_per_gpu": 304.598025, "total_per_gpu": null, "tpot_error_pct": 119.49, "tpot_ms": 16.328, "tpot_relative": 2.194909, "ttft_error_pct": 86.51, "ttft_ms": 498.74, "ttft_relative": 0.134864, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16201.378913697452, "interactivity_tok_s": 69.13359929486181, "output_per_gpu": 298.8073280273693, "total_per_gpu": 2687.717457197086, "tpot_error_pct": 94.44, "tpot_ms": 14.46474666731727, "tpot_relative": 1.944439, "ttft_error_pct": 22.24, "ttft_ms": 2875.5283827772105, "ttft_relative": 0.77757, "unavailable_metrics": {}}, "concurrency": 216, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 10551.148911785842, "interactivity_tok_s": 134.42608991910134, "output_per_gpu": 507.6551799128599, "total_per_gpu": 4110.400304452158, "tpot_ms": 7.439032114984582, "tpot_relative": 1.0, "ttft_ms": 3698.094700722882, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 61.61, "tpot_shape_error_pct": null, "ttft_mape_pct": 94.01, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.29, "tpot_shape_error_pct": null, "ttft_mape_pct": 52.09, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "58ecb0cbf5874009", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 23368.928, "interactivity_tok_s": 44.73072105922347, "output_per_gpu": 1023.4013461538462, "total_per_gpu": null, "tpot_error_pct": 61.61, "tpot_ms": 22.356, "tpot_relative": 1.616052, "ttft_error_pct": 94.01, "ttft_ms": 498.74, "ttft_relative": 0.059866, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21735.382912004363, "interactivity_tok_s": 51.896131865863765, "output_per_gpu": 1372.3824219792111, "total_per_gpu": 12348.500122620391, "tpot_error_pct": 39.29, "tpot_ms": 19.2692588839705, "tpot_relative": 1.39292, "ttft_error_pct": 52.09, "ttft_ms": 3991.0334007025867, "ttft_relative": 0.479064, "unavailable_metrics": {}}, "concurrency": 1728, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 21069.77273935107, "interactivity_tok_s": 72.28718005351156, "output_per_gpu": 2031.935065443771, "total_per_gpu": 12659.254513637015, "tpot_ms": 13.833711582880069, "tpot_relative": 1.0, "ttft_ms": 8330.906001223597, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 52}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 67.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 69.39, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "68a06567cf53dbce", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 40108.81725413064, "interactivity_tok_s": 28.401462386309746, "output_per_gpu": 2348.4309522482718, "total_per_gpu": 21142.22430563286, "tpot_error_pct": 67.33, "tpot_ms": 35.20945458364941, "tpot_relative": 1.673333, "ttft_error_pct": 69.39, "ttft_ms": 7699.190492603611, "ttft_relative": 0.30608, "unavailable_metrics": {}}, "concurrency": 4608, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 44522.27256626329, "interactivity_tok_s": 47.525112826462816, "output_per_gpu": 3846.59501643357, "total_per_gpu": 18891.463289215633, "tpot_ms": 21.041507121750218, "tpot_relative": 1.0, "ttft_ms": 25154.17956141961, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 85.97, "tpot_shape_error_pct": null, "ttft_mape_pct": 97.51, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 50.71, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.65, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "1f79c5e0b88c6569", "is_multinode": true, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 62127.329, "interactivity_tok_s": 16.599438938963864, "output_per_gpu": 1995.6326000000001, "total_per_gpu": null, "tpot_error_pct": 85.97, "tpot_ms": 60.243, "tpot_relative": 1.859746, "ttft_error_pct": 97.51, "ttft_ms": 498.74, "ttft_relative": 0.024946, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 55050.11362338652, "interactivity_tok_s": 20.48321792063048, "output_per_gpu": 2520.650190687871, "total_per_gpu": 22667.749413733123, "tpot_error_pct": 50.71, "tpot_ms": 48.82045408464901, "tpot_relative": 1.507123, "ttft_error_pct": 49.65, "ttft_ms": 10066.18516561057, "ttft_relative": 0.503498, "unavailable_metrics": {}}, "concurrency": 1536, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 49840.93365666956, "interactivity_tok_s": 30.870735463197388, "output_per_gpu": 6893.8333170393025, "total_per_gpu": 24800.96687872541, "tpot_ms": 32.393138193683534, "tpot_relative": 1.0, "ttft_ms": 19992.50512134131, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 10}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 120.1, "tpot_shape_error_pct": null, "ttft_mape_pct": 79.67, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 95.3, "tpot_shape_error_pct": null, "ttft_mape_pct": 9.93, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "d585457fcc0bde99", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13125.629, "interactivity_tok_s": 81.01758081503687, "output_per_gpu": 107.72859259259259, "total_per_gpu": null, "tpot_error_pct": 120.1, "tpot_ms": 12.343, "tpot_relative": 2.200978, "ttft_error_pct": 79.67, "ttft_ms": 498.74, "ttft_relative": 0.203344, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12269.330681251438, "interactivity_tok_s": 91.30261030259803, "output_per_gpu": 105.13865573388784, "total_per_gpu": 949.5672595979189, "tpot_error_pct": 95.3, "tpot_ms": 10.952589380366762, "tpot_relative": 1.953043, "ttft_error_pct": 9.93, "ttft_ms": 2209.112043971935, "ttft_relative": 0.900689, "unavailable_metrics": {}}, "concurrency": 78, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 7604.062869932209, "interactivity_tok_s": 178.31795318870277, "output_per_gpu": 175.9122521377344, "total_per_gpu": 1530.1670446964445, "tpot_ms": 5.607960287328794, "tpot_relative": 1.0, "ttft_ms": 2452.690618888529, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 54}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 122.65, "tpot_shape_error_pct": null, "ttft_mape_pct": 83.41, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 97.92, "tpot_shape_error_pct": null, "ttft_mape_pct": 17.17, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "78c62c4f6697e65b", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 14847.338, "interactivity_tok_s": 71.29616426636247, "output_per_gpu": 184.588796875, "total_per_gpu": null, "tpot_error_pct": 122.65, "tpot_ms": 14.026, "tpot_relative": 2.226517, "ttft_error_pct": 83.41, "ttft_ms": 498.74, "ttft_relative": 0.165864, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13965.379949213317, "interactivity_tok_s": 80.2060147770768, "output_per_gpu": 179.6862716574575, "total_per_gpu": 1617.9323323710312, "tpot_error_pct": 97.92, "tpot_ms": 12.467892872864741, "tpot_relative": 1.97918, "ttft_error_pct": 17.17, "ttft_ms": 2490.750013723981, "ttft_relative": 0.828338, "unavailable_metrics": {}}, "concurrency": 180, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 8805.577801503201, "interactivity_tok_s": 158.74213993588128, "output_per_gpu": 304.1496695434209, "total_per_gpu": 2567.093184817459, "tpot_ms": 6.299524501836232, "tpot_relative": 1.0, "ttft_ms": 3006.9253424924, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.22, "tpot_shape_error_pct": null, "ttft_mape_pct": 16.37, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "e39b7319aac85dca", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 8797.923478244818, "interactivity_tok_s": 127.76588720494662, "output_per_gpu": 73.38412960162704, "total_per_gpu": 662.8138528426396, "tpot_error_pct": 23.22, "tpot_ms": 7.826815293787462, "tpot_relative": 1.232236, "ttft_error_pct": 16.37, "ttft_ms": 1602.6717060690169, "ttft_relative": 1.163671, "unavailable_metrics": {}}, "concurrency": 48, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 7216.231688680087, "interactivity_tok_s": 157.437731657365, "output_per_gpu": 92.67945175856471, "total_per_gpu": 811.7743392923901, "tpot_ms": 6.351717529672753, "tpot_relative": 1.0, "ttft_ms": 1377.2552380248726, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 66}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.17, "tpot_shape_error_pct": null, "ttft_mape_pct": 18.75, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "7a2cda34eec43c6a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 16}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 7056.2562074791595, "interactivity_tok_s": 160.10278867396065, "output_per_gpu": 46.27389226564907, "total_per_gpu": 413.46139477882645, "tpot_error_pct": 26.17, "tpot_ms": 6.245987395237929, "tpot_relative": 1.261709, "ttft_error_pct": 18.75, "ttft_ms": 1269.3460025338509, "ttft_relative": 1.187456, "unavailable_metrics": {}}, "concurrency": 24, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 5655.206704937275, "interactivity_tok_s": 202.00315372018238, "output_per_gpu": 59.925968682070916, "total_per_gpu": 519.3436599127906, "tpot_ms": 4.950417761225719, "tpot_relative": 1.0, "ttft_ms": 1068.9627938644435, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 66}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 62.51, "tpot_shape_error_pct": null, "ttft_mape_pct": 91.5, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 41.93, "tpot_shape_error_pct": null, "ttft_mape_pct": 45.18, "ttft_shape_error_pct": null}, "framework": "dynamo-trt", "id": "601ef5e821dfc5e1", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 18854.429, "interactivity_tok_s": 55.732040349997206, "output_per_gpu": 751.0445483870968, "total_per_gpu": null, "tpot_error_pct": 62.51, "tpot_ms": 17.943, "tpot_relative": 1.625135, "ttft_error_pct": 91.5, "ttft_ms": 498.74, "ttft_relative": 0.084998, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17656.94902717017, "interactivity_tok_s": 63.814956137594066, "output_per_gpu": 946.2253816940936, "total_per_gpu": 8508.88236362269, "tpot_error_pct": 41.93, "tpot_ms": 15.670307722908383, "tpot_relative": 1.419293, "ttft_error_pct": 45.18, "ttft_ms": 3216.832409445295, "ttft_relative": 0.548231, "unavailable_metrics": {}}, "concurrency": 1152, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "31131975660", "measured": {"e2e_ms": 16041.43560094619, "interactivity_tok_s": 90.57209134755267, "output_per_gpu": 1335.0313367766169, "total_per_gpu": 9295.864031978985, "tpot_ms": 11.040928669325917, "tpot_relative": 1.0, "ttft_ms": 5867.662112710816, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 62}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 19}]}, {"aic": {"coverage_pct": 89.66, "points": 26, "status_counts": {"failed": 3, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.58, "tpot_shape_error_pct": 8.5, "ttft_mape_pct": 42.62, "ttft_shape_error_pct": 47.6}, "aisimulate": {"coverage_pct": 100.0, "points": 29, "status_counts": {"failed": 0, "success": 29, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.32, "tpot_shape_error_pct": 4.94, "ttft_mape_pct": 54.96, "ttft_shape_error_pct": 30.17}, "frameworks": ["dynamo-sglang", "sglang"], "gpu_skus": ["b200", "b300", "gb300"], "hf_model_paths": ["nvidia/Qwen3.5-397B-A17B-NVFP4-V2"], "model": "nvidia/Qwen3.5-397B-A17B-NVFP4-V2", "precisions": ["fp4"], "rows": 29, "workloads": [{"aic": {"coverage_pct": 89.66, "points": 26, "status_counts": {"failed": 3, "success": 26, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.58, "tpot_shape_error_pct": 8.5, "ttft_mape_pct": 42.62, "ttft_shape_error_pct": 47.6}, "aisimulate": {"coverage_pct": 100.0, "points": 29, "status_counts": {"failed": 0, "success": 29, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.32, "tpot_shape_error_pct": 4.94, "ttft_mape_pct": 54.96, "ttft_shape_error_pct": 30.17}, "gpu_skus": ["b200", "b300", "gb300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.35, "tpot_shape_error_pct": 1.63, "ttft_mape_pct": 33.87, "ttft_shape_error_pct": 49.29}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.77, "tpot_shape_error_pct": 1.56, "ttft_mape_pct": 44.18, "ttft_shape_error_pct": 25.59}, "gpu": "b200", "precisions": ["fp4"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.95, "tpot_shape_error_pct": null, "ttft_mape_pct": 45.95, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.77, "tpot_shape_error_pct": null, "ttft_mape_pct": 27.19, "ttft_shape_error_pct": null}, "framework": "sglang", "id": "2b26ffeccfc5c8f4", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6448.743721996016, "interactivity_tok_s": 165.34801029543945, "output_per_gpu": 155.99347266181118, "total_per_gpu": null, "tpot_error_pct": 21.95, "tpot_ms": 6.047850217327843, "tpot_relative": 1.219455, "ttft_error_pct": 45.95, "ttft_ms": 261.7929496696324, "ttft_relative": 1.459532, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5818.4209642999795, "interactivity_tok_s": 161.60517006378083, "output_per_gpu": 154.29978181623378, "total_per_gpu": 1387.0630162426307, "tpot_error_pct": 24.77, "tpot_ms": 6.187920841921884, "tpot_relative": 1.247699, "ttft_error_pct": 27.19, "ttft_ms": 130.59054242499968, "ttft_relative": 0.728061, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 4738.204559509177, "interactivity_tok_s": 201.6345391181895, "output_per_gpu": 189.66921675284462, "total_per_gpu": 1705.0131424741858, "tpot_ms": 4.959467779544669, "tpot_relative": 1.0, "ttft_ms": 179.36769993975759, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.75, "tpot_shape_error_pct": 1.63, "ttft_mape_pct": 31.85, "ttft_shape_error_pct": 49.29}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.93, "tpot_shape_error_pct": 1.56, "ttft_mape_pct": 47.01, "ttft_shape_error_pct": 25.59}, "framework": "sglang", "id": "0e00899559355179", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 7833.215940134037, "interactivity_tok_s": 136.61490467686284, "output_per_gpu": 256.4404413291892, "total_per_gpu": null, "tpot_error_pct": 23.14, "tpot_ms": 7.319845534902024, "tpot_relative": 1.231363, "ttft_error_pct": 36.03, "ttft_ms": 345.0139579292667, "ttft_relative": 1.360273, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7069.896924025001, "interactivity_tok_s": 133.23156025203397, "output_per_gpu": 254.52472343515527, "total_per_gpu": 2288.025468608567, "tpot_error_pct": 26.26, "tpot_ms": 7.505729108841038, "tpot_relative": 1.262633, "ttft_error_pct": 32.65, "ttft_ms": 170.8268811750001, "ttft_relative": 0.673512, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 5716.736849129666, "interactivity_tok_s": 168.22256986988842, "output_per_gpu": 314.8593170486452, "total_per_gpu": 2830.397482465141, "tpot_ms": 5.9445055486517, "tpot_relative": 1.0, "ttft_ms": 253.63590767374262, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9779.331087529823, "interactivity_tok_s": 108.82387180745157, "output_per_gpu": 413.867341694606, "total_per_gpu": null, "tpot_error_pct": 22.94, "tpot_ms": 9.189160276978184, "tpot_relative": 1.545824, "ttft_error_pct": 5.37, "ttft_ms": 378.82012418113953, "ttft_relative": 1.493559, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8840.822669774992, "interactivity_tok_s": 107.20027375173214, "output_per_gpu": 412.586767066743, "total_per_gpu": 3670.1500480822137, "tpot_error_pct": 24.8, "tpot_ms": 9.32833438761477, "tpot_relative": 1.569236, "ttft_error_pct": 49.41, "ttft_ms": 202.5120436250004, "ttft_relative": 0.798436, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 7321.079118049238, "interactivity_tok_s": 133.79015190132756, "output_per_gpu": 498.1914409005205, "total_per_gpu": 4431.643200227533, "tpot_ms": 7.474391693175717, "tpot_relative": 1.257361, "ttft_ms": 400.30433460487984, "ttft_relative": 1.578264, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13516.009612144202, "interactivity_tok_s": 78.27667833901893, "output_per_gpu": 604.007764420514, "total_per_gpu": null, "tpot_error_pct": 23.29, "tpot_ms": 12.775197175191394, "tpot_relative": 2.149076, "ttft_error_pct": 12.03, "ttft_ms": 446.98290192340653, "ttft_relative": 1.762301, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12006.511642868743, "interactivity_tok_s": 77.92162227685893, "output_per_gpu": 594.3104115005757, "total_per_gpu": 5363.502709633041, "tpot_error_pct": 23.85, "tpot_ms": 12.833408376008347, "tpot_relative": 2.158869, "ttft_error_pct": 48.01, "ttft_ms": 264.15447124375, "ttft_relative": 1.041471, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 9993.255741870962, "interactivity_tok_s": 96.50457654106258, "output_per_gpu": 713.8526950116234, "total_per_gpu": 6442.3418972359395, "tpot_ms": 10.362202869979967, "tpot_relative": 1.743156, "ttft_ms": 508.1151180376764, "ttft_relative": 2.003325, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20064.988951053787, "interactivity_tok_s": 52.5151860518185, "output_per_gpu": 820.42595189131, "total_per_gpu": null, "tpot_error_pct": 25.77, "tpot_ms": 19.042110962213222, "tpot_relative": 3.203313, "ttft_error_pct": 25.44, "ttft_ms": 584.9094367096612, "ttft_relative": 2.306099, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18016.990071043838, "interactivity_tok_s": 52.4774131859735, "output_per_gpu": 805.4817375876161, "total_per_gpu": 7202.874168520656, "tpot_error_pct": 25.86, "tpot_ms": 19.055817337187012, "tpot_relative": 3.205619, "ttft_error_pct": 51.49, "ttft_ms": 380.5099711468748, "ttft_relative": 1.500221, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 14802.015358480276, "interactivity_tok_s": 66.04936001459406, "output_per_gpu": 978.8463555393517, "total_per_gpu": 8753.155782750526, "tpot_ms": 15.140192119636636, "tpot_relative": 2.546922, "ttft_ms": 784.4728440162726, "ttft_relative": 3.092909, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31591.32698890445, "interactivity_tok_s": 33.102700892979904, "output_per_gpu": 1043.90974407441, "total_per_gpu": null, "tpot_error_pct": 29.76, "tpot_ms": 30.20901536805023, "tpot_relative": 5.081838, "ttft_error_pct": 45.86, "ttft_ms": 687.5042673890648, "ttft_relative": 2.710595, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28555.681410410907, "interactivity_tok_s": 32.995221491504495, "output_per_gpu": 1018.3420117841098, "total_per_gpu": 9179.108797772711, "tpot_error_pct": 30.19, "tpot_ms": 30.307418916932466, "tpot_relative": 5.098392, "ttft_error_pct": 51.77, "ttft_ms": 612.4370755203124, "ttft_relative": 2.414631, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 22741.198541868653, "interactivity_tok_s": 42.95492782021216, "output_per_gpu": 1276.8638999891032, "total_per_gpu": 11509.367699967888, "tpot_ms": 23.280216048447333, "tpot_relative": 3.916258, "ttft_ms": 1269.760976532416, "ttft_relative": 5.006235, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 47713.39898836412, "interactivity_tok_s": 21.760563016071043, "output_per_gpu": 1379.9251215198296, "total_per_gpu": null, "tpot_error_pct": 23.58, "tpot_ms": 45.954693325786664, "tpot_relative": 7.730617, "ttft_error_pct": 66.4, "ttft_ms": 701.7477160843621, "ttft_relative": 2.766752, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43739.04877648185, "interactivity_tok_s": 21.578010361541665, "output_per_gpu": 1329.2156954192078, "total_per_gpu": 12000.268088125627, "tpot_error_pct": 24.63, "tpot_ms": 46.343475753552, "tpot_relative": 7.796019, "ttft_error_pct": 48.72, "ttft_ms": 1070.89489945156, "ttft_relative": 4.222174, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34452191302", "measured": {"e2e_ms": 36339.92949924286, "interactivity_tok_s": 26.89271174815625, "output_per_gpu": 1599.7936035144744, "total_per_gpu": 14443.06758790386, "tpot_ms": 37.18479599100152, "tpot_relative": 6.255322, "ttft_ms": 2088.3895179402316, "ttft_relative": 8.233809, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.48, "tpot_shape_error_pct": 7.5, "ttft_mape_pct": 31.35, "ttft_shape_error_pct": 40.66}, "aisimulate": {"coverage_pct": 100.0, "points": 12, "status_counts": {"failed": 0, "success": 12, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.34, "tpot_shape_error_pct": 5.86, "ttft_mape_pct": 50.63, "ttft_shape_error_pct": 13.22}, "gpu": "b300", "precisions": ["fp4"], "rows": 12, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.06, "tpot_shape_error_pct": 9.5, "ttft_mape_pct": 31.73, "ttft_shape_error_pct": 43.23}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.95, "tpot_shape_error_pct": 7.39, "ttft_mape_pct": 48.18, "ttft_shape_error_pct": 16.59}, "framework": "sglang", "id": "c69280d3a61be7ae", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 7652.621882364991, "interactivity_tok_s": 139.70328578260558, "output_per_gpu": 262.60349585994396, "total_per_gpu": null, "tpot_error_pct": 11.24, "tpot_ms": 7.158027775782706, "tpot_relative": 1.112444, "ttft_error_pct": 22.14, "ttft_ms": 329.95946773928216, "ttft_relative": 1.221407, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6900.604503274952, "interactivity_tok_s": 136.41321993709877, "output_per_gpu": 260.8103163464485, "total_per_gpu": 2344.529200239212, "tpot_error_pct": 13.93, "tpot_ms": 7.330667808157509, "tpot_relative": 1.139274, "ttft_error_pct": 39.86, "ttft_ms": 162.46452197499997, "ttft_relative": 0.601393, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 6184.019220218761, "interactivity_tok_s": 155.41203221739303, "output_per_gpu": 290.67982903043867, "total_per_gpu": 2613.0383054983236, "tpot_ms": 6.434508227787555, "tpot_relative": 1.0, "ttft_ms": 270.14704309985973, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9842.808426590016, "interactivity_tok_s": 107.9060979114128, "output_per_gpu": 411.46077586998985, "total_per_gpu": null, "tpot_error_pct": 15.72, "tpot_ms": 9.267316855632808, "tpot_relative": 1.440253, "ttft_error_pct": 7.47, "ttft_ms": 362.34328327765274, "ttft_relative": 1.341282, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8861.543669050016, "interactivity_tok_s": 106.82153830633168, "output_per_gpu": 411.7688186679258, "total_per_gpu": 3662.874018905151, "tpot_error_pct": 16.89, "tpot_ms": 9.361407969358241, "tpot_relative": 1.454875, "ttft_error_pct": 42.85, "ttft_ms": 192.689808725, "ttft_relative": 0.713278, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 7750.916553026764, "interactivity_tok_s": 124.86548849426744, "output_per_gpu": 470.431487204063, "total_per_gpu": 4184.705577583586, "tpot_ms": 8.008618010139045, "tpot_relative": 1.244636, "ttft_ms": 337.1487659373088, "ttft_relative": 1.248019, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13539.295310749058, "interactivity_tok_s": 78.02039656161726, "output_per_gpu": 603.0905276407678, "total_per_gpu": null, "tpot_error_pct": 18.73, "tpot_ms": 12.817161204893923, "tpot_relative": 1.991941, "ttft_error_pct": 18.58, "ttft_ms": 427.3393981425745, "ttft_relative": 1.581877, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12017.091700324987, "interactivity_tok_s": 77.76490354007214, "output_per_gpu": 594.0699493609558, "total_per_gpu": 5361.332599009943, "tpot_error_pct": 19.12, "tpot_ms": 12.859271399786428, "tpot_relative": 1.998486, "ttft_error_pct": 52.06, "ttft_ms": 251.63188371875032, "ttft_relative": 0.931463, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 10409.937045589322, "interactivity_tok_s": 92.63555983816127, "output_per_gpu": 685.2357787246985, "total_per_gpu": 6184.081390476983, "tpot_ms": 10.794990625058537, "tpot_relative": 1.677671, "ttft_ms": 524.8793203965761, "ttft_relative": 1.942939, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19113.15440732621, "interactivity_tok_s": 55.137387327530796, "output_per_gpu": 861.2967535121622, "total_per_gpu": null, "tpot_error_pct": 18.38, "tpot_ms": 18.136514050978388, "tpot_relative": 2.818633, "ttft_error_pct": 30.03, "ttft_ms": 559.5005331753164, "ttft_relative": 2.071096, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17139.29070799994, "interactivity_tok_s": 55.16818934163319, "output_per_gpu": 846.141455327216, "total_per_gpu": 7566.466310886329, "tpot_error_pct": 18.31, "tpot_ms": 18.126387904584366, "tpot_relative": 2.817059, "ttft_error_pct": 54.63, "ttft_ms": 362.8155369343742, "ttft_relative": 1.34303, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 14981.668563839776, "interactivity_tok_s": 65.27038440506372, "output_per_gpu": 966.3830628511616, "total_per_gpu": 8641.70505113327, "tpot_ms": 15.320884182235938, "tpot_relative": 2.38105, "ttft_ms": 799.630727434851, "ttft_relative": 2.959983, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30711.84477566549, "interactivity_tok_s": 34.03871775810687, "output_per_gpu": 1073.711750676034, "total_per_gpu": null, "tpot_error_pct": 29.98, "tpot_ms": 29.378309932424933, "tpot_relative": 4.565743, "ttft_error_pct": 47.32, "ttft_ms": 657.8337147947815, "ttft_relative": 2.435095, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27723.389268387527, "interactivity_tok_s": 33.97201409666405, "output_per_gpu": 1048.7350463426635, "total_per_gpu": 9453.06486329795, "tpot_error_pct": 30.24, "tpot_ms": 29.435993908238636, "tpot_relative": 4.574708, "ttft_error_pct": 53.2, "ttft_ms": 584.4338815609378, "ttft_relative": 2.163392, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 22091.700550800306, "interactivity_tok_s": 44.243822363740264, "output_per_gpu": 1313.0853168021783, "total_per_gpu": 11835.859509094169, "tpot_ms": 22.602025471007757, "tpot_relative": 3.512627, "ttft_ms": 1248.8319735847654, "ttft_relative": 4.622786, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 46249.59661007526, "interactivity_tok_s": 22.44502454164749, "output_per_gpu": 1423.5334027545873, "total_per_gpu": null, "tpot_error_pct": 26.28, "tpot_ms": 44.55330392463892, "tpot_relative": 6.92412, "ttft_error_pct": 64.83, "ttft_ms": 671.5666951696511, "ttft_relative": 2.48593, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 42338.96873673143, "interactivity_tok_s": 22.28314467783528, "output_per_gpu": 1372.8657186075282, "total_per_gpu": 12394.344070013236, "tpot_error_pct": 27.2, "tpot_ms": 44.876969317292335, "tpot_relative": 6.974421, "ttft_error_pct": 46.46, "ttft_ms": 1022.4026554234364, "ttft_relative": 3.784615, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 34399.60162311319, "interactivity_tok_s": 28.344052648194573, "output_per_gpu": 1685.2197814203644, "total_per_gpu": 15214.302113758062, "tpot_ms": 35.28076991713099, "tpot_relative": 5.483056, "ttft_ms": 1909.466380468723, "ttft_relative": 7.068248, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.9, "tpot_shape_error_pct": 5.5, "ttft_mape_pct": 30.97, "ttft_shape_error_pct": 38.08}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.73, "tpot_shape_error_pct": 4.33, "ttft_mape_pct": 53.08, "ttft_shape_error_pct": 9.86}, "framework": "sglang", "id": "ed7b36ebd59969ee", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 6468.739380220399, "interactivity_tok_s": 164.6892147157554, "output_per_gpu": 155.5725283413161, "total_per_gpu": null, "tpot_error_pct": 11.87, "tpot_ms": 6.072043040134385, "tpot_relative": 1.118728, "ttft_error_pct": 2.19, "ttft_ms": 257.0393501629233, "ttft_relative": 1.021949, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5829.15806357499, "interactivity_tok_s": 161.2422143728729, "output_per_gpu": 154.14714518593638, "total_per_gpu": 1385.6909039666616, "tpot_error_pct": 14.26, "tpot_ms": 6.201849831257578, "tpot_relative": 1.142643, "ttft_error_pct": 48.88, "ttft_ms": 128.56729965000022, "ttft_relative": 0.511164, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 5239.523375686258, "interactivity_tok_s": 184.24236783454305, "output_per_gpu": 171.63591487484152, "total_per_gpu": 1542.9045134062135, "tpot_ms": 5.42763324067806, "tpot_relative": 1.0, "ttft_ms": 251.5187936834991, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8025.155657874556, "interactivity_tok_s": 132.1208366371172, "output_per_gpu": 252.46482518301633, "total_per_gpu": null, "tpot_error_pct": 11.6, "tpot_ms": 7.5688288498096465, "tpot_relative": 1.394499, "ttft_error_pct": 9.28, "ttft_ms": 282.2437445192871, "ttft_relative": 1.122158, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7253.40903785, "interactivity_tok_s": 130.39127077532657, "output_per_gpu": 251.32267795430232, "total_per_gpu": 2235.631416721905, "tpot_error_pct": 13.08, "tpot_ms": 7.669225048991746, "tpot_relative": 1.412996, "ttft_error_pct": 51.12, "ttft_ms": 152.07618856250033, "ttft_relative": 0.604632, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 6587.996300510713, "interactivity_tok_s": 147.44048010052052, "output_per_gpu": 276.6153089434554, "total_per_gpu": 2460.621063144446, "tpot_ms": 6.782397882306337, "tpot_relative": 1.249605, "ttft_ms": 311.1168064555386, "ttft_relative": 1.236953, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10617.234930499675, "interactivity_tok_s": 99.47232013028433, "output_per_gpu": 384.5465081150686, "total_per_gpu": null, "tpot_error_pct": 13.2, "tpot_ms": 10.053047910114547, "tpot_relative": 1.852197, "ttft_error_pct": 24.05, "ttft_ms": 332.9669184524929, "ttft_relative": 1.323825, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9418.205403200036, "interactivity_tok_s": 99.22525904377001, "output_per_gpu": 378.11640816307795, "total_per_gpu": 3412.405942239525, "tpot_error_pct": 13.49, "tpot_ms": 10.078079005658049, "tpot_relative": 1.856809, "ttft_error_pct": 55.04, "ttft_ms": 197.1220057874997, "ttft_relative": 0.783727, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 8569.714593950357, "interactivity_tok_s": 112.6063683668504, "output_per_gpu": 415.6305820594912, "total_per_gpu": 3750.9619719665297, "tpot_ms": 8.880492413556823, "tpot_relative": 1.636163, "ttft_ms": 438.40306743113615, "ttft_relative": 1.743023, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14780.622871598414, "interactivity_tok_s": 71.31886481793718, "output_per_gpu": 556.9001341223654, "total_per_gpu": null, "tpot_error_pct": 17.19, "tpot_ms": 14.021535572008897, "tpot_relative": 2.583361, "ttft_error_pct": 32.41, "ttft_ms": 436.5919814333117, "ttft_relative": 1.735823, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13254.187105031224, "interactivity_tok_s": 71.35279904114381, "output_per_gpu": 546.3511755885634, "total_per_gpu": 4885.646174143949, "tpot_error_pct": 17.14, "tpot_ms": 14.01486715921789, "tpot_relative": 2.582132, "ttft_error_pct": 56.27, "ttft_ms": 282.4571086124997, "ttft_relative": 1.123006, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 11721.948638784852, "interactivity_tok_s": 83.58081845633649, "output_per_gpu": 616.7390871435364, "total_per_gpu": 5515.077291271274, "tpot_ms": 11.964467666973261, "tpot_relative": 2.204362, "ttft_ms": 645.9430671551672, "ttft_relative": 2.56817, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23258.502010392316, "interactivity_tok_s": 44.97898278685182, "output_per_gpu": 709.0195453225507, "total_per_gpu": null, "tpot_error_pct": 24.69, "tpot_ms": 22.23260594262079, "tpot_relative": 4.096188, "ttft_error_pct": 51.51, "ttft_ms": 514.5461310912473, "ttft_relative": 2.045756, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20971.51390234221, "interactivity_tok_s": 44.93647453360414, "output_per_gpu": 692.7002976546944, "total_per_gpu": 6243.846686912458, "tpot_error_pct": 24.81, "tpot_ms": 22.253637170672693, "tpot_relative": 4.100063, "ttft_error_pct": 57.31, "ttft_ms": 452.977559464062, "ttft_relative": 1.800969, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 17502.362997783348, "interactivity_tok_s": 56.08607320760034, "output_per_gpu": 828.4503216829588, "total_per_gpu": 7467.46726372891, "tpot_ms": 17.829738165097424, "tpot_relative": 3.284993, "ttft_ms": 1061.0571732366225, "ttft_relative": 4.2186, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35506.16177746164, "interactivity_tok_s": 29.248958770429468, "output_per_gpu": 927.2303251912996, "total_per_gpu": null, "tpot_error_pct": 22.86, "tpot_ms": 34.18925124305602, "tpot_relative": 6.299109, "ttft_error_pct": 66.37, "ttft_ms": 530.5577558153308, "ttft_relative": 2.109416, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32454.934429431967, "interactivity_tok_s": 29.077793814010054, "output_per_gpu": 895.3224487505239, "total_per_gpu": 8083.044345135369, "tpot_error_pct": 23.59, "tpot_ms": 34.390504534019605, "tpot_relative": 6.336188, "ttft_error_pct": 49.87, "ttft_ms": 790.861138021875, "ttft_relative": 3.144342, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "30506346629", "measured": {"e2e_ms": 27202.7654222029, "interactivity_tok_s": 35.93602345293309, "output_per_gpu": 1066.4062163859144, "total_per_gpu": 9627.602601727265, "tpot_ms": 27.82723028077221, "tpot_relative": 5.126955, "ttft_ms": 1577.7106872239528, "ttft_relative": 6.272735, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 70.0, "points": 7, "status_counts": {"failed": 3, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.42, "tpot_shape_error_pct": 15.89, "ttft_mape_pct": 70.69, "ttft_shape_error_pct": 57.76}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.79, "tpot_shape_error_pct": 6.21, "ttft_mape_pct": 67.72, "ttft_shape_error_pct": 62.23}, "gpu": "gb300", "precisions": ["fp4"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 216.49, "tpot_shape_error_pct": null, "ttft_mape_pct": 84.92, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "567de03309662be3", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 67212.61950493576, "interactivity_tok_s": 14.640086019154483, "output_per_gpu": 1557.2078037408028, "total_per_gpu": 14019.104498517472, "tpot_error_pct": 216.49, "tpot_ms": 68.30560959079348, "tpot_relative": 3.164911, "ttft_error_pct": 84.92, "ttft_ms": 4351.54430983966, "ttft_relative": 0.150766, "unavailable_metrics": {}}, "concurrency": 5120, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 48724.00005917428, "interactivity_tok_s": 46.334564075405595, "output_per_gpu": 5610.632957401358, "total_per_gpu": 18370.362940164516, "tpot_ms": 21.58216053079909, "tpot_relative": 1.0, "ttft_ms": 28862.87896434851, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 44}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 243.3, "tpot_shape_error_pct": null, "ttft_mape_pct": 79.95, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "caafa699a5e0fec1", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 68250.7033732347, "interactivity_tok_s": 15.253881465679175, "output_per_gpu": 1687.143599480161, "total_per_gpu": 15188.879973693092, "tpot_error_pct": 243.3, "tpot_ms": 65.55708474921437, "tpot_relative": 3.43298, "ttft_error_pct": 79.95, "ttft_ms": 7917.340881393241, "ttft_relative": 0.2005, "unavailable_metrics": {}}, "concurrency": 5120, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 57061.75946017706, "interactivity_tok_s": 52.366271565959146, "output_per_gpu": 4803.754317266566, "total_per_gpu": 17301.32447603307, "tpot_ms": 19.096261202030146, "tpot_relative": 1.0, "ttft_ms": 39487.94235201922, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.42, "tpot_shape_error_pct": 15.89, "ttft_mape_pct": 70.69, "ttft_shape_error_pct": 57.76}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.33, "tpot_shape_error_pct": 6.21, "ttft_mape_pct": 61.65, "ttft_shape_error_pct": 62.23}, "framework": "dynamo-sglang", "id": "b0ae937f1f88d6af", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5804.786, "interactivity_tok_s": 183.55359765051395, "output_per_gpu": 21.079, "total_per_gpu": null, "tpot_error_pct": 30.22, "tpot_ms": 5.448, "tpot_relative": 1.302159, "ttft_error_pct": 41.95, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4730.333436399984, "interactivity_tok_s": 201.2267403720673, "output_per_gpu": 24.583362159032177, "total_per_gpu": 220.0584829792071, "tpot_error_pct": 18.78, "tpot_ms": 4.9695184554051055, "tpot_relative": 1.187794, "ttft_error_pct": 71.93, "ttft_ms": 111.92398609999898, "ttft_relative": 0.28066, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 4286.5562605671585, "interactivity_tok_s": 239.01598399069698, "output_per_gpu": 54.25046662018912, "total_per_gpu": 242.87041102844637, "tpot_ms": 4.18382061025225, "tpot_relative": 1.0, "ttft_ms": 398.7883179448545, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6553.622, "interactivity_tok_s": 161.81229773462783, "output_per_gpu": 74.542125, "total_per_gpu": null, "tpot_error_pct": 27.35, "tpot_ms": 6.18, "tpot_relative": 1.477119, "ttft_error_pct": 53.25, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5298.377653250014, "interactivity_tok_s": 178.21494330997248, "output_per_gpu": 84.6205418100914, "total_per_gpu": 760.6882043357639, "tpot_error_pct": 15.62, "tpot_ms": 5.611201740028511, "tpot_relative": 1.341167, "ttft_error_pct": 71.54, "ttft_ms": 140.92422115000142, "ttft_relative": 0.353381, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 4955.213484854903, "interactivity_tok_s": 206.05997978495546, "output_per_gpu": 179.4461945334751, "total_per_gpu": 806.7521585101968, "tpot_ms": 4.852955925957101, "tpot_relative": 1.159934, "ttft_ms": 495.10669806040823, "ttft_relative": 1.241528, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7666.646, "interactivity_tok_s": 137.58943313153551, "output_per_gpu": 126.70975, "total_per_gpu": null, "tpot_error_pct": 28.47, "tpot_ms": 7.268, "tpot_relative": 1.737168, "ttft_error_pct": 68.93, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6265.924288537508, "interactivity_tok_s": 152.08150220548123, "output_per_gpu": 145.0335300576209, "total_per_gpu": 1290.1403045444808, "tpot_error_pct": 16.22, "tpot_ms": 6.575421635754717, "tpot_relative": 1.571631, "ttft_error_pct": 75.33, "ttft_ms": 183.83276197499944, "ttft_relative": 0.460978, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 5976.9295822654385, "interactivity_tok_s": 176.75616645503482, "output_per_gpu": 302.2379304743359, "total_per_gpu": 1344.5995033276092, "tpot_ms": 5.657511248720089, "tpot_relative": 1.352236, "ttft_ms": 745.1405305415392, "ttft_relative": 1.868511, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9098.846, "interactivity_tok_s": 115.36686663590218, "output_per_gpu": 212.439, "total_per_gpu": null, "tpot_error_pct": 29.5, "tpot_ms": 8.668, "tpot_relative": 2.071791, "ttft_error_pct": 71.89, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7326.953748493707, "interactivity_tok_s": 128.7551190430304, "output_per_gpu": 241.6283259614885, "total_per_gpu": 2180.63516293839, "tpot_error_pct": 16.03, "tpot_ms": 7.766681491442656, "tpot_relative": 1.856361, "ttft_error_pct": 72.46, "ttft_ms": 226.79449683124977, "ttft_relative": 0.568709, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 6942.666474188445, "interactivity_tok_s": 149.3977061924908, "output_per_gpu": 507.8915076365852, "total_per_gpu": 2292.351840454804, "tpot_ms": 6.693543197454146, "tpot_relative": 1.599864, "ttft_ms": 823.4462924592663, "ttft_relative": 2.064871, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11577.575, "interactivity_tok_s": 90.1631953836444, "output_per_gpu": 332.08325, "total_per_gpu": null, "tpot_error_pct": 31.65, "tpot_ms": 11.091, "tpot_relative": 2.650926, "ttft_error_pct": 77.53, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9500.984100796926, "interactivity_tok_s": 100.98456094658745, "output_per_gpu": 377.7040996839962, "total_per_gpu": 3377.550322998204, "tpot_error_pct": 17.54, "tpot_ms": 9.90250381470607, "tpot_relative": 2.366857, "ttft_error_pct": 66.14, "ttft_ms": 348.84535415937626, "ttft_relative": 0.874763, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 8816.568086843472, "interactivity_tok_s": 118.70145601527102, "output_per_gpu": 811.4740100017148, "total_per_gpu": 3629.106333803251, "tpot_ms": 8.424496493719078, "tpot_relative": 2.013589, "ttft_ms": 1030.3311732655857, "ttft_relative": 2.583654, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16619.942, "interactivity_tok_s": 62.421972534332085, "output_per_gpu": 459.85275, "total_per_gpu": null, "tpot_error_pct": 50.15, "tpot_ms": 16.02, "tpot_relative": 3.829036, "ttft_error_pct": 82.77, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13524.493655454697, "interactivity_tok_s": 70.75764500532931, "output_per_gpu": 530.8495214602837, "total_per_gpu": 4784.959725065874, "tpot_error_pct": 32.46, "tpot_ms": 14.132748481449351, "tpot_relative": 3.377953, "ttft_error_pct": 61.66, "ttft_ms": 515.2969416250014, "ttft_relative": 1.292157, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 11165.20272658745, "interactivity_tok_s": 93.72571977537591, "output_per_gpu": 1274.412869532741, "total_per_gpu": 5745.0199128367885, "tpot_ms": 10.669429932323924, "tpot_relative": 2.550164, "ttft_ms": 1343.8529095961712, "ttft_relative": 3.36984, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32036.552, "interactivity_tok_s": 32.1646831778707, "output_per_gpu": 895.79525, "total_per_gpu": null, "tpot_error_pct": 127.63, "tpot_ms": 31.09, "tpot_relative": 7.431007, "ttft_error_pct": 98.51, "ttft_ms": 231.482, "ttft_relative": 0.580463, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 35000.01149422467, "interactivity_tok_s": 52.41812325537889, "output_per_gpu": 811.5345206236525, "total_per_gpu": 7301.710728209776, "tpot_error_pct": 39.68, "tpot_ms": 19.077371296336615, "tpot_relative": 4.559797, "ttft_error_pct": 12.46, "ttft_ms": 17429.818351754937, "ttft_relative": 43.706943, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 28076.80181684591, "interactivity_tok_s": 73.21656114640221, "output_per_gpu": 2020.7288660548293, "total_per_gpu": 9091.453293507653, "tpot_ms": 13.658112103905319, "tpot_relative": 3.264507, "ttft_ms": 15498.135508167434, "ttft_relative": 38.863063, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 111.8, "tpot_shape_error_pct": null, "ttft_mape_pct": 80.77, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "3bdf569cd545515a", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 34983.90705536459, "interactivity_tok_s": 28.337340626064748, "output_per_gpu": 1457.5388872686367, "total_per_gpu": 13105.944592682457, "tpot_error_pct": 111.8, "tpot_ms": 35.28912656963293, "tpot_relative": 2.118015, "ttft_error_pct": 80.77, "ttft_ms": 2466.926515815331, "ttft_relative": 0.192252, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29473335540", "measured": {"e2e_ms": 28184.386353132322, "interactivity_tok_s": 60.018903803849824, "output_per_gpu": 3909.490558149071, "total_per_gpu": 15626.023760277325, "tpot_ms": 16.661417263936375, "tpot_relative": 1.0, "ttft_ms": 12831.710984158803, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 36}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 29}]}, {"aic": {"coverage_pct": 100.0, "points": 207, "status_counts": {"failed": 0, "success": 207, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.45, "tpot_shape_error_pct": 17.57, "ttft_mape_pct": 43.1, "ttft_shape_error_pct": 38.43}, "aisimulate": {"coverage_pct": 100.0, "points": 207, "status_counts": {"failed": 0, "success": 207, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.22, "tpot_shape_error_pct": 14.92, "ttft_mape_pct": 49.06, "ttft_shape_error_pct": 30.23}, "frameworks": ["trt", "vllm"], "gpu_skus": ["b200", "h100", "h200"], "hf_model_paths": ["openai/gpt-oss-120b"], "model": "openai/gpt-oss-120b", "precisions": ["fp4"], "rows": 207, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 96, "status_counts": {"failed": 0, "success": 96, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.91, "tpot_shape_error_pct": 24.64, "ttft_mape_pct": 47.2, "ttft_shape_error_pct": 32.84}, "aisimulate": {"coverage_pct": 100.0, "points": 96, "status_counts": {"failed": 0, "success": 96, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.22, "tpot_shape_error_pct": 19.18, "ttft_mape_pct": 51.97, "ttft_shape_error_pct": 24.07}, "gpu_skus": ["b200", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 42, "status_counts": {"failed": 0, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.57, "tpot_shape_error_pct": 11.3, "ttft_mape_pct": 52.1, "ttft_shape_error_pct": 26.36}, "aisimulate": {"coverage_pct": 100.0, "points": 42, "status_counts": {"failed": 0, "success": 42, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.65, "tpot_shape_error_pct": 11.88, "ttft_mape_pct": 42.27, "ttft_shape_error_pct": 18.19}, "gpu": "b200", "precisions": ["fp4"], "rows": 42, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.8, "tpot_shape_error_pct": null, "ttft_mape_pct": 70.37, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.37, "tpot_shape_error_pct": null, "ttft_mape_pct": 77.87, "ttft_shape_error_pct": null}, "framework": "trt", "id": "6dde3748dc55b961", "is_multinode": false, "parallelism": {"attention_dp_size": 4, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 5355.906385727323, "interactivity_tok_s": 192.77365993582876, "output_per_gpu": 3058.2816858595406, "total_per_gpu": null, "tpot_error_pct": 0.8, "tpot_ms": 5.187430691168513, "tpot_relative": 0.991969, "ttft_error_pct": 70.37, "ttft_ms": 49.164788661934594, "ttft_relative": 0.296277, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5548.202316231253, "interactivity_tok_s": 167.1960101993708, "output_per_gpu": 2574.8035838842716, "total_per_gpu": 5150.768088157031, "tpot_error_pct": 14.37, "tpot_ms": 5.981003965391055, "tpot_relative": 1.14372, "ttft_error_pct": 77.87, "ttft_ms": 36.72235552031255, "ttft_relative": 0.221297, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 4982.196566695711, "interactivity_tok_s": 191.2254917527645, "output_per_gpu": 2851.571194443906, "total_per_gpu": 5704.428097498588, "tpot_ms": 5.2294283091341205, "tpot_relative": 1.0, "ttft_ms": 165.94180535576015, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.06, "tpot_shape_error_pct": 14.81, "ttft_mape_pct": 44.15, "ttft_shape_error_pct": 13.16}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.64, "tpot_shape_error_pct": 12.33, "ttft_mape_pct": 59.2, "ttft_shape_error_pct": 26.23}, "framework": "trt", "id": "dac3b876a41df471", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 3014.3072803422942, "interactivity_tok_s": 342.52016586585853, "output_per_gpu": 677.198983652949, "total_per_gpu": null, "tpot_error_pct": 3.48, "tpot_ms": 2.9195361314627846, "tpot_relative": 1.034843, "ttft_error_pct": 42.83, "ttft_ms": 27.621817855865533, "ttft_relative": 0.571728, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2708.6277359499954, "interactivity_tok_s": 340.4878911423725, "output_per_gpu": 657.375682947874, "total_per_gpu": 1321.1802490082896, "tpot_error_pct": 4.1, "tpot_ms": 2.9369620066220135, "tpot_relative": 1.04102, "ttft_error_pct": 66.69, "ttft_ms": 16.09247497499975, "ttft_relative": 0.333089, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2634.157617308665, "interactivity_tok_s": 354.4545942117063, "output_per_gpu": 676.8048071432216, "total_per_gpu": 1360.2285068131919, "tpot_ms": 2.821235826337538, "tpot_relative": 1.0, "ttft_ms": 48.31286990083754, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3589.2604472001767, "interactivity_tok_s": 287.44912106187223, "output_per_gpu": 1139.0631455836037, "total_per_gpu": null, "tpot_error_pct": 1.45, "tpot_ms": 3.4788765271080933, "tpot_relative": 1.233104, "ttft_error_pct": 39.3, "ttft_ms": 30.36975996859767, "ttft_relative": 0.628606, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3265.5667122125046, "interactivity_tok_s": 285.91500658993067, "output_per_gpu": 1115.7079366900941, "total_per_gpu": 2223.174831789858, "tpot_error_pct": 1.99, "tpot_ms": 3.497542895445971, "tpot_relative": 1.23972, "ttft_error_pct": 62.85, "ttft_ms": 18.590420225000102, "ttft_relative": 0.384792, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 3234.062063262718, "interactivity_tok_s": 291.61647029421147, "output_per_gpu": 1127.0522436585384, "total_per_gpu": 2245.7796523768334, "tpot_ms": 3.429161593620214, "tpot_relative": 1.215482, "ttft_ms": 50.03589815023588, "ttft_relative": 1.035664, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4113.337210068524, "interactivity_tok_s": 250.89138359194862, "output_per_gpu": 1991.1905332669567, "total_per_gpu": null, "tpot_error_pct": 5.95, "tpot_ms": 3.9857885339992642, "tpot_relative": 1.412781, "ttft_error_pct": 41.64, "ttft_ms": 35.87553978727682, "ttft_relative": 0.742567, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3708.346602187521, "interactivity_tok_s": 248.24735605852888, "output_per_gpu": 1933.5086198735223, "total_per_gpu": 3887.6000956118946, "tpot_error_pct": 4.95, "tpot_ms": 4.028240283712152, "tpot_relative": 1.427828, "ttft_error_pct": 64.35, "ttft_ms": 21.916406275000174, "ttft_relative": 0.453635, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 3940.65200160112, "interactivity_tok_s": 235.9659042044546, "output_per_gpu": 1825.018765058932, "total_per_gpu": 3669.4654746358005, "tpot_ms": 4.2379004007864705, "tpot_relative": 1.502143, "ttft_ms": 61.46908490045462, "ttft_relative": 1.272313, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5048.742902595799, "interactivity_tok_s": 204.5243543420621, "output_per_gpu": 3252.2344069680316, "total_per_gpu": null, "tpot_error_pct": 10.44, "tpot_ms": 4.889393261828975, "tpot_relative": 1.733068, "ttft_error_pct": 37.73, "ttft_ms": 46.8935957447585, "ttft_relative": 0.970623, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4619.6838018812505, "interactivity_tok_s": 200.95613449936795, "output_per_gpu": 3123.6634640976563, "total_per_gpu": 6237.445963793232, "tpot_error_pct": 8.85, "tpot_ms": 4.976210367955427, "tpot_relative": 1.763841, "ttft_error_pct": 64.18, "ttft_ms": 26.975798796874948, "ttft_relative": 0.558356, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 5116.559112977848, "interactivity_tok_s": 183.16700122263657, "output_per_gpu": 2827.357183952678, "total_per_gpu": 5645.770697754041, "tpot_ms": 5.459498672386496, "tpot_relative": 1.935144, "ttft_ms": 75.31092280405574, "ttft_relative": 1.558817, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6736.40754418262, "interactivity_tok_s": 153.1117672501499, "output_per_gpu": 4878.985383177425, "total_per_gpu": null, "tpot_error_pct": 10.29, "tpot_ms": 6.531176655849231, "tpot_relative": 2.315006, "ttft_error_pct": 34.72, "ttft_ms": 55.013825248855795, "ttft_relative": 1.138699, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6260.677883246864, "interactivity_tok_s": 148.01661944412504, "output_per_gpu": 4595.238874470408, "total_per_gpu": 9192.549637659915, "tpot_error_pct": 7.21, "tpot_ms": 6.755998101804312, "tpot_relative": 2.394695, "ttft_error_pct": 56.44, "ttft_ms": 36.704586168750055, "ttft_relative": 0.759727, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 6793.790174237074, "interactivity_tok_s": 137.35169410790039, "output_per_gpu": 4245.418201392127, "total_per_gpu": 8492.750565316304, "tpot_ms": 7.28058002120034, "tpot_relative": 2.580635, "ttft_ms": 84.26808239491947, "ttft_relative": 1.744216, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8314.07320725981, "interactivity_tok_s": 123.88663910755774, "output_per_gpu": 7903.620658705142, "total_per_gpu": null, "tpot_error_pct": 21.16, "tpot_ms": 8.071895461881125, "tpot_relative": 2.86112, "ttft_error_pct": 49.13, "ttft_ms": 56.5241497554177, "ttft_relative": 1.169961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7926.21420686013, "interactivity_tok_s": 116.7958741297695, "output_per_gpu": 7246.802191361743, "total_per_gpu": 14509.524441613596, "tpot_error_pct": 16.37, "tpot_ms": 8.561946279788279, "tpot_relative": 3.034821, "ttft_error_pct": 52.0, "ttft_ms": 53.32513192421858, "ttft_relative": 1.103746, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 9528.878429423276, "interactivity_tok_s": 97.67220150473652, "output_per_gpu": 6055.349201701425, "total_per_gpu": 12124.001031699532, "tpot_ms": 10.238327636666467, "tpot_relative": 3.629022, "ttft_ms": 111.10496407600294, "ttft_relative": 2.299697, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11248.13310909137, "interactivity_tok_s": 91.44246789254274, "output_per_gpu": 11679.885630206614, "total_per_gpu": null, "tpot_error_pct": 24.65, "tpot_ms": 10.935837833851282, "tpot_relative": 3.876258, "ttft_error_pct": 63.74, "ttft_ms": 60.77100506150764, "ttft_relative": 1.257864, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11191.210349083969, "interactivity_tok_s": 83.03534304796666, "output_per_gpu": 10320.044514996292, "total_per_gpu": 20630.114740442397, "tpot_error_pct": 17.02, "tpot_ms": 12.04306459506447, "tpot_relative": 4.26872, "ttft_error_pct": 47.91, "ttft_ms": 87.30124039453092, "ttft_relative": 1.806998, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 13553.707804195983, "interactivity_tok_s": 68.90203029728124, "output_per_gpu": 8546.991986607418, "total_per_gpu": 17085.723333182355, "tpot_ms": 14.513360429082425, "tpot_relative": 5.144327, "ttft_ms": 167.59310493898738, "ttft_relative": 3.468912, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.59, "tpot_shape_error_pct": 4.62, "ttft_mape_pct": 14.52, "ttft_shape_error_pct": 16.34}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.52, "tpot_shape_error_pct": 6.31, "ttft_mape_pct": 39.13, "ttft_shape_error_pct": 8.85}, "framework": "trt", "id": "4da9e078aa690b66", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 3433.9401848177176, "interactivity_tok_s": 301.4511387109935, "output_per_gpu": 1187.4613518850406, "total_per_gpu": null, "tpot_error_pct": 10.66, "tpot_ms": 3.3172871871574436, "tpot_relative": 0.893447, "ttft_error_pct": 4.38, "ttft_ms": 40.35539235565287, "ttft_relative": 1.043816, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3078.0773172999934, "interactivity_tok_s": 300.0595979325068, "output_per_gpu": 1160.9811015392247, "total_per_gpu": 2333.3161548464786, "tpot_error_pct": 10.24, "tpot_ms": 3.3326712656094832, "tpot_relative": 0.89759, "ttft_error_pct": 41.29, "ttft_ms": 22.69734659999977, "ttft_relative": 0.58708, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 3442.9701506276615, "interactivity_tok_s": 269.3305512922544, "output_per_gpu": 1036.8530754990677, "total_per_gpu": 2083.8461780788016, "tpot_ms": 3.712909639110662, "tpot_relative": 1.0, "ttft_ms": 38.661390758352354, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4548.173251675492, "interactivity_tok_s": 227.14322919296782, "output_per_gpu": 1797.0341364310068, "total_per_gpu": null, "tpot_error_pct": 7.91, "tpot_ms": 4.402508512153173, "tpot_relative": 1.18573, "ttft_error_pct": 9.87, "ttft_ms": 44.40704374279583, "ttft_relative": 1.148615, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4137.0343979000045, "interactivity_tok_s": 225.82738445370973, "output_per_gpu": 1765.9285845256984, "total_per_gpu": 3518.8133513711155, "tpot_error_pct": 7.38, "tpot_ms": 4.428160926625711, "tpot_relative": 1.192639, "ttft_error_pct": 36.45, "ttft_ms": 25.685707037499906, "ttft_relative": 0.664376, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 4478.564893631847, "interactivity_tok_s": 209.1716913231451, "output_per_gpu": 1629.6534983613453, "total_per_gpu": 3247.2697584668936, "tpot_ms": 4.780761649314774, "tpot_relative": 1.287605, "ttft_ms": 40.41777977836318, "ttft_relative": 1.04543, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5783.672558535502, "interactivity_tok_s": 178.49860112628076, "output_per_gpu": 2832.1246676820924, "total_per_gpu": null, "tpot_error_pct": 8.5, "tpot_ms": 5.6022848004984604, "tpot_relative": 1.508866, "ttft_error_pct": 8.37, "ttft_ms": 52.53520762557698, "ttft_relative": 1.358855, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5202.4176615374945, "interactivity_tok_s": 176.9689999126552, "output_per_gpu": 2763.331810894391, "total_per_gpu": 5556.080227324326, "tpot_error_pct": 7.71, "tpot_ms": 5.650707188793291, "tpot_relative": 1.521908, "ttft_error_pct": 36.25, "ttft_ms": 30.904028462499838, "ttft_relative": 0.799351, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 5653.754409740213, "interactivity_tok_s": 163.32374946287632, "output_per_gpu": 2539.3326528210205, "total_per_gpu": 5105.697364071313, "tpot_ms": 6.1228082461289635, "tpot_relative": 1.649059, "ttft_ms": 48.47556774766417, "ttft_relative": 1.25385, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7145.808112870318, "interactivity_tok_s": 144.5600553121237, "output_per_gpu": 4595.847608758944, "total_per_gpu": null, "tpot_error_pct": 16.26, "tpot_ms": 6.917540241949077, "tpot_relative": 1.863105, "ttft_error_pct": 1.5, "ttft_ms": 69.16444535641241, "ttft_relative": 1.78898, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6526.185459265602, "interactivity_tok_s": 142.24286153414008, "output_per_gpu": 4433.573887234949, "total_per_gpu": 8853.123220846477, "tpot_error_pct": 14.89, "tpot_ms": 7.030229771917147, "tpot_relative": 1.893456, "ttft_error_pct": 46.27, "ttft_ms": 37.727775775000055, "ttft_relative": 0.975851, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 7693.743264266232, "interactivity_tok_s": 121.05731883480722, "output_per_gpu": 3762.28005199987, "total_per_gpu": 7512.659028326339, "tpot_ms": 8.260549710047545, "tpot_relative": 2.224818, "ttft_ms": 70.21631688476191, "ttft_relative": 1.816187, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16476.18238322982, "interactivity_tok_s": 62.448258127886305, "output_per_gpu": 15948.843728732967, "total_per_gpu": null, "tpot_error_pct": 4.64, "tpot_ms": 16.013256894245533, "tpot_relative": 4.312859, "ttft_error_pct": 48.47, "ttft_ms": 94.62058041664096, "ttft_relative": 2.447418, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15967.309928767, "interactivity_tok_s": 58.17888968673608, "output_per_gpu": 14458.028989333232, "total_per_gpu": 28902.08434054358, "tpot_error_pct": 2.36, "tpot_ms": 17.188365150735855, "tpot_relative": 4.629352, "ttft_error_pct": 35.37, "ttft_ms": 118.67202939140557, "ttft_relative": 3.069523, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 15661.18470533445, "interactivity_tok_s": 59.5517367318181, "output_per_gpu": 14724.897843231372, "total_per_gpu": 29435.564120458173, "tpot_ms": 16.792121521213446, "tpot_relative": 4.522631, "ttft_ms": 183.61345084202867, "ttft_relative": 4.749272, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.87, "tpot_shape_error_pct": null, "ttft_mape_pct": 70.27, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.63, "tpot_shape_error_pct": null, "ttft_mape_pct": 66.81, "ttft_shape_error_pct": null}, "framework": "trt", "id": "f38591d11d84501e", "is_multinode": false, "parallelism": {"attention_dp_size": 2, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 10312.14705137377, "interactivity_tok_s": 99.91530702768054, "output_per_gpu": 12745.47826474393, "total_per_gpu": null, "tpot_error_pct": 9.87, "tpot_ms": 10.008476476211598, "tpot_relative": 0.901338, "ttft_error_pct": 70.27, "ttft_ms": 73.47561620930523, "ttft_relative": 0.297343, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11103.59831708635, "interactivity_tok_s": 83.67410923041332, "output_per_gpu": 10366.374550011375, "total_per_gpu": 20722.730032643994, "tpot_error_pct": 7.63, "tpot_ms": 11.95112812311274, "tpot_relative": 1.076288, "ttft_error_pct": 66.81, "ttft_ms": 82.00934523398449, "ttft_relative": 0.331877, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 10477.617790308796, "interactivity_tok_s": 90.05741721231095, "output_per_gpu": 10911.396924590734, "total_per_gpu": 21812.248019445487, "tpot_ms": 11.104027085770108, "tpot_relative": 1.0, "ttft_ms": 247.1074982335267, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 1, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.53, "tpot_shape_error_pct": 18.92, "ttft_mape_pct": 49.09, "ttft_shape_error_pct": 62.68}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.22, "tpot_shape_error_pct": 17.75, "ttft_mape_pct": 70.37, "ttft_shape_error_pct": 60.74}, "framework": "trt", "id": "64e2dc3cf1b14cdb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 2376.6950626317816, "interactivity_tok_s": 434.16000063722294, "output_per_gpu": 214.77592875384855, "total_per_gpu": null, "tpot_error_pct": 0.4, "tpot_ms": 2.303298319818236, "tpot_relative": 0.995969, "ttft_error_pct": 65.37, "ttft_ms": 20.420881457725745, "ttft_relative": 0.346332, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2137.0939441999985, "interactivity_tok_s": 431.40376415399453, "output_per_gpu": 207.9666558100653, "total_per_gpu": 417.9671460867686, "tpot_error_pct": 0.23, "tpot_ms": 2.3180140812193715, "tpot_relative": 1.002332, "ttft_error_pct": 79.64, "ttft_ms": 12.003933699999838, "ttft_relative": 0.203583, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2179.288987745531, "interactivity_tok_s": 432.4097452888917, "output_per_gpu": 204.15791151901962, "total_per_gpu": 410.31240944015667, "tpot_ms": 2.3126213294103786, "tpot_relative": 1.0, "ttft_ms": 58.963354578008875, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 2886.171701055259, "interactivity_tok_s": 357.72801078911624, "output_per_gpu": 709.4128472414587, "total_per_gpu": null, "tpot_error_pct": 14.75, "tpot_ms": 2.795419899588206, "tpot_relative": 1.208767, "ttft_error_pct": 41.98, "ttft_ms": 26.457143776524134, "ttft_relative": 0.448705, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2604.553082337521, "interactivity_tok_s": 353.47572831513105, "output_per_gpu": 687.2081622490224, "total_per_gpu": 1381.731888756375, "tpot_error_pct": 13.73, "tpot_ms": 2.8290485594769863, "tpot_relative": 1.223308, "ttft_error_pct": 65.78, "ttft_ms": 15.605720006250172, "ttft_relative": 0.264668, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 3046.9217729143566, "interactivity_tok_s": 304.9566741456463, "output_per_gpu": 588.5909561360633, "total_per_gpu": 1183.4476628816558, "tpot_ms": 3.279154334961049, "tpot_relative": 1.417938, "ttft_ms": 45.60045826510759, "ttft_relative": 0.773369, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3330.3632155638966, "interactivity_tok_s": 310.39319923262116, "output_per_gpu": 1232.747674177391, "total_per_gpu": null, "tpot_error_pct": 24.04, "tpot_ms": 3.2217200714199916, "tpot_relative": 1.393103, "ttft_error_pct": 50.83, "ttft_ms": 34.54358250124522, "ttft_relative": 0.585848, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3044.6162792062287, "interactivity_tok_s": 305.01702935274676, "output_per_gpu": 1181.2068444274578, "total_per_gpu": 2358.6772227101374, "tpot_error_pct": 22.71, "tpot_ms": 3.2785054727010596, "tpot_relative": 1.417658, "ttft_error_pct": 73.55, "ttft_ms": 18.57880652187513, "ttft_relative": 0.315091, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 3986.810059104755, "interactivity_tok_s": 235.7607769207078, "output_per_gpu": 906.0064192888964, "total_per_gpu": 1809.1469033450305, "tpot_ms": 4.241587651097387, "tpot_relative": 1.834104, "ttft_ms": 70.25232707019313, "ttft_relative": 1.191457, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4220.599664559881, "interactivity_tok_s": 244.7270484565285, "output_per_gpu": 1947.4960407904168, "total_per_gpu": null, "tpot_error_pct": 18.93, "tpot_ms": 4.086185022484887, "tpot_relative": 1.766906, "ttft_error_pct": 38.17, "ttft_ms": 40.432386557840786, "ttft_relative": 0.685721, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3914.738450940616, "interactivity_tok_s": 236.83869180023288, "output_per_gpu": 1829.6043416483067, "total_per_gpu": 3660.0336102915817, "tpot_error_pct": 16.23, "tpot_ms": 4.222283075450667, "tpot_relative": 1.825756, "ttft_error_pct": 62.5, "ttft_ms": 24.525141081249966, "ttft_relative": 0.415939, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 4710.775077314611, "interactivity_tok_s": 198.38858680057135, "output_per_gpu": 1524.9955509316126, "total_per_gpu": 3050.6786876808433, "tpot_ms": 5.040612547964982, "tpot_relative": 2.17961, "ttft_ms": 65.3918623120262, "ttft_relative": 1.109025, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.1, "tpot_shape_error_pct": 14.45, "ttft_mape_pct": 53.5, "ttft_shape_error_pct": 15.87}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.37, "tpot_shape_error_pct": 10.09, "ttft_mape_pct": 65.24, "ttft_shape_error_pct": 75.7}, "framework": "trt", "id": "f8dc341b407bac1c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 2561.31533077308, "interactivity_tok_s": 402.8707025183263, "output_per_gpu": 398.5873411382934, "total_per_gpu": null, "tpot_error_pct": 0.94, "tpot_ms": 2.482185956310662, "tpot_relative": 0.990558, "ttft_error_pct": 56.92, "ttft_ms": 22.039097467272647, "ttft_relative": 0.430799, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2302.298817849998, "interactivity_tok_s": 400.44290240246715, "output_per_gpu": 386.56848568626236, "total_per_gpu": 776.9174635232608, "tpot_error_pct": 0.34, "tpot_ms": 2.4972349216342082, "tpot_relative": 0.996564, "ttft_error_pct": 74.79, "ttft_ms": 12.898394450000035, "ttft_relative": 0.252125, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2348.565279180184, "interactivity_tok_s": 399.06679451258935, "output_per_gpu": 379.5601951313853, "total_per_gpu": 762.8323440085051, "tpot_ms": 2.5058461734992914, "tpot_relative": 1.0, "ttft_ms": 51.1586994107347, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6123.134613078794, "interactivity_tok_s": 168.31267470497266, "output_per_gpu": 5366.591051793911, "total_per_gpu": null, "tpot_error_pct": 15.26, "tpot_ms": 5.941323205473698, "tpot_relative": 2.370985, "ttft_error_pct": 50.08, "ttft_ms": 45.160973879200874, "ttft_relative": 0.882762, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5817.170134450048, "interactivity_tok_s": 159.17846973969347, "output_per_gpu": 4930.115799925324, "total_per_gpu": 9871.062271338194, "tpot_error_pct": 10.4, "tpot_ms": 6.2822566496292644, "tpot_relative": 2.50704, "ttft_error_pct": 55.7, "ttft_ms": 40.079857574218835, "ttft_relative": 0.783442, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19841791766", "measured": {"e2e_ms": 6538.909574128047, "interactivity_tok_s": 142.63156441676287, "output_per_gpu": 4393.563235796016, "total_per_gpu": 8796.778423391499, "tpot_ms": 7.011070824954609, "tpot_relative": 2.797886, "ttft_ms": 90.47511680855678, "ttft_relative": 1.768519, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.45, "tpot_shape_error_pct": 9.35, "ttft_mape_pct": 48.71, "ttft_shape_error_pct": 25.43}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.71, "tpot_shape_error_pct": 8.34, "ttft_mape_pct": 51.03, "ttft_shape_error_pct": 10.36}, "framework": "vllm", "id": "765ba6758aae297c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 2718.2363395586135, "interactivity_tok_s": 383.493978875586, "output_per_gpu": 188.17348313541316, "total_per_gpu": null, "tpot_error_pct": 8.26, "tpot_ms": 2.607602870146815, "tpot_relative": 1.082645, "ttft_error_pct": 86.69, "ttft_ms": 50.65860339842189, "ttft_relative": 1.866943, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2391.2343342000095, "interactivity_tok_s": 385.6659562377159, "output_per_gpu": 185.90259312094653, "total_per_gpu": 373.6232426021976, "tpot_error_pct": 7.65, "tpot_ms": 2.5929174816343457, "tpot_relative": 1.076548, "ttft_error_pct": 47.99, "ttft_ms": 14.111774825000191, "ttft_relative": 0.520067, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2235.13435596833, "interactivity_tok_s": 415.18802867947045, "output_per_gpu": 197.1968701341113, "total_per_gpu": 396.32225034417525, "tpot_ms": 2.4085472868294344, "tpot_relative": 1.0, "ttft_ms": 27.134520339313895, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3143.4827952848113, "interactivity_tok_s": 331.3109676599598, "output_per_gpu": 325.4352152124034, "total_per_gpu": null, "tpot_error_pct": 14.32, "tpot_ms": 3.018312394132233, "tpot_relative": 1.253167, "ttft_error_pct": 71.52, "ttft_ms": 55.749216087536986, "ttft_relative": 2.05455, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2814.642803574995, "interactivity_tok_s": 331.7117952826356, "output_per_gpu": 323.2211139429947, "total_per_gpu": 644.0547942617569, "tpot_error_pct": 14.18, "tpot_ms": 3.014665182912619, "tpot_relative": 1.251653, "ttft_error_pct": 50.61, "ttft_ms": 16.053281574999982, "ttft_relative": 0.591618, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2483.2317458407488, "interactivity_tok_s": 378.75216089798005, "output_per_gpu": 365.7269297644615, "total_per_gpu": 728.7524618425052, "tpot_ms": 2.6402489628814503, "tpot_relative": 1.0962, "ttft_ms": 32.50304865650833, "ttft_relative": 1.197849, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3609.182833967079, "interactivity_tok_s": 288.30536434823836, "output_per_gpu": 566.8873243950108, "total_per_gpu": null, "tpot_error_pct": 11.21, "tpot_ms": 3.4685445491472704, "tpot_relative": 1.440098, "ttft_error_pct": 65.27, "ttft_ms": 60.86176018942151, "ttft_relative": 2.242964, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3206.096589593756, "interactivity_tok_s": 287.1092579701713, "output_per_gpu": 558.996109112859, "total_per_gpu": 1123.9429216384779, "tpot_error_pct": 10.84, "tpot_ms": 3.482994616996618, "tpot_relative": 1.446098, "ttft_error_pct": 49.19, "ttft_ms": 18.71243304375, "ttft_relative": 0.689617, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 3611.4828975347336, "interactivity_tok_s": 255.99342379842102, "output_per_gpu": 495.79917042697673, "total_per_gpu": 996.8762913931541, "tpot_ms": 3.90635034745048, "tpot_relative": 1.62187, "ttft_ms": 36.82628066162579, "ttft_relative": 1.357175, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4646.336229807104, "interactivity_tok_s": 223.3449812320017, "output_per_gpu": 880.6939053934722, "total_per_gpu": null, "tpot_error_pct": 3.43, "tpot_ms": 4.477378423655916, "tpot_relative": 1.858954, "ttft_error_pct": 3.81, "ttft_ms": 65.97810240710167, "ttft_relative": 2.431519, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4211.019049334358, "interactivity_tok_s": 220.40615809518593, "output_per_gpu": 857.4634680682809, "total_per_gpu": 1712.2145549529139, "tpot_error_pct": 2.15, "tpot_ms": 4.537078313248099, "tpot_relative": 1.883741, "ttft_error_pct": 62.54, "ttft_ms": 23.80962998124993, "ttft_relative": 0.877466, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 4342.714788480953, "interactivity_tok_s": 215.67836356214565, "output_per_gpu": 827.4944741150047, "total_per_gpu": 1652.371366811388, "tpot_ms": 4.636533695285849, "tpot_relative": 1.925033, "ttft_ms": 63.55551794695202, "ttft_relative": 2.342238, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6270.4930228016365, "interactivity_tok_s": 165.01854521156358, "output_per_gpu": 1305.1605304782581, "total_per_gpu": null, "tpot_error_pct": 5.01, "tpot_ms": 6.059924953997992, "tpot_relative": 2.516008, "ttft_error_pct": 16.26, "ttft_ms": 71.18979486169063, "ttft_relative": 2.623588, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5814.671211515627, "interactivity_tok_s": 159.359557690446, "output_per_gpu": 1238.092571676338, "total_per_gpu": 2476.7433711407257, "tpot_error_pct": 8.73, "tpot_ms": 6.275117818427232, "tpot_relative": 2.605354, "ttft_error_pct": 44.8, "ttft_ms": 33.80261104999994, "ttft_relative": 1.245742, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 5377.486106797733, "interactivity_tok_s": 173.2789071605876, "output_per_gpu": 1330.2200080712812, "total_per_gpu": 2661.039782096832, "tpot_ms": 5.771042860244046, "tpot_relative": 2.396068, "ttft_ms": 61.234428633906646, "ttft_relative": 2.256698, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.76, "tpot_shape_error_pct": 5.24, "ttft_mape_pct": 59.95, "ttft_shape_error_pct": 24.45}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.67, "tpot_shape_error_pct": 7.03, "ttft_mape_pct": 40.12, "ttft_shape_error_pct": 6.77}, "framework": "vllm", "id": "f86cdfefccd4af50", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 2987.4170354100206, "interactivity_tok_s": 348.7766932933881, "output_per_gpu": 342.4362878949688, "total_per_gpu": null, "tpot_error_pct": 13.02, "tpot_ms": 2.8671640600675348, "tpot_relative": 1.130248, "ttft_error_pct": 98.84, "ttft_ms": 54.30820196093242, "ttft_relative": 1.988394, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2630.2612632250084, "interactivity_tok_s": 350.85862734569247, "output_per_gpu": 338.7829404196284, "total_per_gpu": 680.8790486076691, "tpot_error_pct": 12.35, "tpot_ms": 2.850150807364142, "tpot_relative": 1.123541, "ttft_error_pct": 36.69, "ttft_ms": 17.29117012500019, "ttft_relative": 0.633084, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2352.849938190775, "interactivity_tok_s": 394.2040294242061, "output_per_gpu": 375.51407992388016, "total_per_gpu": 754.7005441320687, "tpot_ms": 2.536757428534278, "tpot_relative": 1.0, "ttft_ms": 27.31259780121036, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3439.4167377100302, "interactivity_tok_s": 302.6922067726908, "output_per_gpu": 594.8683035607457, "total_per_gpu": null, "tpot_error_pct": 17.75, "tpot_ms": 3.3036859807591883, "tpot_relative": 1.302326, "ttft_error_pct": 76.84, "ttft_ms": 59.74597939338065, "ttft_relative": 2.187488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3079.690368262495, "interactivity_tok_s": 303.38797401558884, "output_per_gpu": 591.3039467735698, "total_per_gpu": 1178.2402985362592, "tpot_error_pct": 17.48, "tpot_ms": 3.2961095549180124, "tpot_relative": 1.29934, "ttft_error_pct": 41.63, "ttft_ms": 19.718422562499978, "ttft_relative": 0.721953, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2638.091770297615, "interactivity_tok_s": 356.4236307403847, "output_per_gpu": 689.165014903034, "total_per_gpu": 1373.2395958639513, "tpot_ms": 2.805650113385411, "tpot_relative": 1.105999, "ttft_ms": 33.78458869410679, "ttft_relative": 1.23696, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3911.882870230178, "interactivity_tok_s": 265.94331616233427, "output_per_gpu": 1046.0435896842748, "total_per_gpu": null, "tpot_error_pct": 17.84, "tpot_ms": 3.7601997840381545, "tpot_relative": 1.482286, "ttft_error_pct": 66.23, "ttft_ms": 65.19849115914617, "ttft_relative": 2.387122, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3476.403672450001, "interactivity_tok_s": 264.9917346277509, "output_per_gpu": 1031.0484788860679, "total_per_gpu": 2073.072818966886, "tpot_error_pct": 18.26, "tpot_ms": 3.7737026077615496, "tpot_relative": 1.487609, "ttft_error_pct": 41.56, "ttft_ms": 22.921636368749926, "ttft_relative": 0.839233, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2958.9858137063857, "interactivity_tok_s": 313.38203978809327, "output_per_gpu": 1210.3363043808304, "total_per_gpu": 2433.557049743721, "tpot_ms": 3.190993334130421, "tpot_relative": 1.257902, "ttft_ms": 39.222132218128536, "ttft_relative": 1.436045, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4694.360111116222, "interactivity_tok_s": 221.2539159174587, "output_per_gpu": 1743.3685968446111, "total_per_gpu": null, "tpot_error_pct": 14.83, "tpot_ms": 4.519694016954988, "tpot_relative": 1.781682, "ttft_error_pct": 41.83, "ttft_ms": 70.71313177126889, "ttft_relative": 2.58903, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4248.564975581253, "interactivity_tok_s": 218.72491460611957, "output_per_gpu": 1697.2089336252438, "total_per_gpu": 3389.049151558547, "tpot_error_pct": 16.15, "tpot_ms": 4.571952865089936, "tpot_relative": 1.802282, "ttft_error_pct": 42.29, "ttft_ms": 28.771672715624856, "ttft_relative": 1.053421, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3682.5546200110693, "interactivity_tok_s": 254.05987729110882, "output_per_gpu": 1943.9508207562835, "total_per_gpu": 3881.752416706483, "tpot_ms": 3.936079992883616, "tpot_relative": 1.551619, "ttft_ms": 49.857656701351516, "ttft_relative": 1.825445, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6244.219675471878, "interactivity_tok_s": 165.85770847332947, "output_per_gpu": 2621.304318343519, "total_per_gpu": null, "tpot_error_pct": 25.38, "tpot_ms": 6.029264537685348, "tpot_relative": 2.37676, "ttft_error_pct": 15.99, "ttft_ms": 76.2820534197672, "ttft_relative": 2.792926, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5760.83989812188, "interactivity_tok_s": 161.0706945766655, "output_per_gpu": 2496.6680649160035, "total_per_gpu": 4994.4618206916875, "tpot_error_pct": 29.1, "tpot_ms": 6.208454012247559, "tpot_relative": 2.447398, "ttft_error_pct": 38.43, "ttft_ms": 40.49384736406249, "ttft_relative": 1.482607, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 4496.232390459045, "interactivity_tok_s": 207.94666608414104, "output_per_gpu": 3170.836236248251, "total_per_gpu": 6343.102130454981, "tpot_ms": 4.808925378949677, "tpot_relative": 1.895698, "ttft_ms": 65.76686801781761, "ttft_relative": 2.407932, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.9, "tpot_shape_error_pct": 13.14, "ttft_mape_pct": 71.94, "ttft_shape_error_pct": 35.13}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.36, "tpot_shape_error_pct": 16.48, "ttft_mape_pct": 12.06, "ttft_shape_error_pct": 6.39}, "framework": "vllm", "id": "62eb0f6c56837400", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 4326.524857881028, "interactivity_tok_s": 241.01112959785414, "output_per_gpu": 945.7937107529091, "total_per_gpu": null, "tpot_error_pct": 20.69, "tpot_ms": 4.149185980201736, "tpot_relative": 1.206917, "ttft_error_pct": 138.82, "ttft_ms": 81.90760013465203, "ttft_relative": 2.388182, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3791.742443224989, "interactivity_tok_s": 244.40708142739166, "output_per_gpu": 944.6555916974764, "total_per_gpu": 1898.5495542963502, "tpot_error_pct": 19.01, "tpot_ms": 4.091534476659914, "tpot_relative": 1.190147, "ttft_error_pct": 18.12, "ttft_ms": 40.51000337499979, "ttft_relative": 1.181151, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3184.90239570383, "interactivity_tok_s": 290.8804401872355, "output_per_gpu": 1120.3644225854264, "total_per_gpu": 2251.6855813311454, "tpot_ms": 3.4378385819146677, "tpot_relative": 1.0, "ttft_ms": 34.297046856954694, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5842.017566831372, "interactivity_tok_s": 177.85426424960588, "output_per_gpu": 1400.885893679174, "total_per_gpu": null, "tpot_error_pct": 34.47, "tpot_ms": 5.62258096098596, "tpot_relative": 1.635499, "ttft_error_pct": 117.2, "ttft_ms": 90.11724374273484, "ttft_relative": 2.627551, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5216.781545187501, "interactivity_tok_s": 179.62152846809064, "output_per_gpu": 1402.1458831210032, "total_per_gpu": 2793.93498544076, "tpot_error_pct": 33.15, "tpot_ms": 5.567261388590443, "tpot_relative": 1.619407, "ttft_error_pct": 14.02, "ttft_ms": 47.30883916249978, "ttft_relative": 1.379385, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3922.7751893748064, "interactivity_tok_s": 239.15823299061339, "output_per_gpu": 1851.2491587639126, "total_per_gpu": 3688.824289756098, "tpot_ms": 4.181332114287901, "tpot_relative": 1.216268, "ttft_ms": 41.490623232675716, "ttft_relative": 1.209743, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7404.3169677071955, "interactivity_tok_s": 140.02166101089847, "output_per_gpu": 2210.6022839630646, "total_per_gpu": null, "tpot_error_pct": 28.07, "tpot_ms": 7.1417521602044545, "tpot_relative": 2.077396, "ttft_error_pct": 93.03, "ttft_ms": 98.30450781803876, "ttft_relative": 2.866267, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6576.546884768926, "interactivity_tok_s": 140.36258642039692, "output_per_gpu": 2187.2706062534016, "total_per_gpu": 4397.825450892501, "tpot_error_pct": 27.76, "tpot_ms": 7.12440562333984, "tpot_relative": 2.07235, "ttft_error_pct": 9.83, "ttft_ms": 55.93299703125022, "ttft_relative": 1.63084, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 5155.500347117777, "interactivity_tok_s": 179.33159183963673, "output_per_gpu": 2776.4558565158413, "total_per_gpu": 5582.468028489703, "tpot_ms": 5.576262329139573, "tpot_relative": 1.622026, "ttft_ms": 50.927156780380756, "ttft_relative": 1.484885, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9864.827624180078, "interactivity_tok_s": 104.83462750782809, "output_per_gpu": 3318.4563630650237, "total_per_gpu": null, "tpot_error_pct": 29.64, "tpot_ms": 9.538832957891982, "tpot_relative": 2.77466, "ttft_error_pct": 48.14, "ttft_ms": 106.60150825658042, "ttft_relative": 3.108183, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8926.952396115681, "interactivity_tok_s": 104.22925876793809, "output_per_gpu": 3248.4963849000874, "total_per_gpu": 6486.716926224703, "tpot_error_pct": 30.39, "tpot_ms": 9.594234976058464, "tpot_relative": 2.790775, "ttft_error_pct": 0.72, "ttft_ms": 71.44284257187537, "ttft_relative": 2.083061, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 6862.905768701239, "interactivity_tok_s": 135.90932589120905, "output_per_gpu": 4200.498925431943, "total_per_gpu": 8387.710574295774, "tpot_ms": 7.3578468103098915, "tpot_relative": 2.140254, "ttft_ms": 71.95979504394927, "ttft_relative": 2.098134, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14607.23943872697, "interactivity_tok_s": 70.59003103178932, "output_per_gpu": 4482.161073256558, "total_per_gpu": null, "tpot_error_pct": 44.02, "tpot_ms": 14.166306281260347, "tpot_relative": 4.120701, "ttft_error_pct": 25.34, "ttft_ms": 115.10811299763475, "ttft_relative": 3.356211, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13473.45755639841, "interactivity_tok_s": 68.91117424552074, "output_per_gpu": 4299.680493325515, "total_per_gpu": 8601.299614816664, "tpot_error_pct": 47.53, "tpot_ms": 14.511434625060108, "tpot_relative": 4.221092, "ttft_error_pct": 11.76, "ttft_ms": 102.64236130781237, "ttft_relative": 2.992746, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 9152.207017793808, "interactivity_tok_s": 101.66514689584983, "output_per_gpu": 6297.852409890776, "total_per_gpu": 12598.544378228687, "tpot_ms": 9.836212611038109, "tpot_relative": 2.861162, "ttft_ms": 91.8388615384174, "ttft_relative": 2.677748, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19019.08364154042, "interactivity_tok_s": 54.14079261267576, "output_per_gpu": 6884.874290893776, "total_per_gpu": null, "tpot_error_pct": 46.52, "tpot_ms": 18.4703612884654, "tpot_relative": 5.372667, "ttft_error_pct": 9.13, "ttft_ms": 123.90404344031234, "ttft_relative": 3.612674, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18047.860447504754, "interactivity_tok_s": 51.41281259052968, "output_per_gpu": 6394.924813267599, "total_per_gpu": 12803.898247835557, "tpot_error_pct": 54.3, "tpot_ms": 19.450404473382214, "tpot_relative": 5.657742, "ttft_error_pct": 17.92, "ttft_ms": 160.78196116718732, "ttft_relative": 4.687924, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 11724.661344448396, "interactivity_tok_s": 79.32812738112209, "output_per_gpu": 9801.819426107611, "total_per_gpu": 19625.171873039202, "tpot_ms": 12.605869229657026, "tpot_relative": 3.666801, "ttft_ms": 136.35375156063674, "ttft_relative": 3.97567, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.18, "tpot_shape_error_pct": 11.77, "ttft_mape_pct": 64.6, "ttft_shape_error_pct": 24.01}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.94, "tpot_shape_error_pct": 14.72, "ttft_mape_pct": 13.45, "ttft_shape_error_pct": 6.2}, "framework": "vllm", "id": "1eb841623061ab90", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 3652.2990951998654, "interactivity_tok_s": 285.1659273276902, "output_per_gpu": 560.1950844302461, "total_per_gpu": null, "tpot_error_pct": 20.37, "tpot_ms": 3.506730307407585, "tpot_relative": 1.203698, "ttft_error_pct": 103.35, "ttft_ms": 64.91399072190582, "ttft_relative": 2.033451, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3212.659992450004, "interactivity_tok_s": 287.7242964302587, "output_per_gpu": 555.6679246981411, "total_per_gpu": 1116.7700694776418, "tpot_error_pct": 19.3, "tpot_ms": 3.4755493797597636, "tpot_relative": 1.192995, "ttft_error_pct": 17.7, "ttft_ms": 26.272435950000148, "ttft_relative": 0.822992, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2702.9292207793333, "interactivity_tok_s": 343.25378188043203, "output_per_gpu": 658.6847464764313, "total_per_gpu": 1323.8111782067524, "tpot_ms": 2.9132963794943327, "tpot_relative": 1.0, "ttft_ms": 31.92305840784684, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4418.517087161262, "interactivity_tok_s": 235.32932562924847, "output_per_gpu": 926.1025632083642, "total_per_gpu": null, "tpot_error_pct": 26.67, "tpot_ms": 4.249364150966286, "tpot_relative": 1.45861, "ttft_error_pct": 102.76, "ttft_ms": 71.41756072275155, "ttft_relative": 2.237178, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3959.910859937504, "interactivity_tok_s": 236.25520682283982, "output_per_gpu": 921.0547503399805, "total_per_gpu": 1835.3062412830252, "tpot_error_pct": 26.18, "tpot_ms": 4.232710946133212, "tpot_relative": 1.452894, "ttft_error_pct": 14.43, "ttft_ms": 30.139632775000177, "ttft_relative": 0.944134, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3149.001819314435, "interactivity_tok_s": 298.10312915904325, "output_per_gpu": 1148.7796349807854, "total_per_gpu": 2289.0739482759636, "tpot_ms": 3.354543787651697, "tpot_relative": 1.15146, "ttft_ms": 35.22272435366176, "ttft_relative": 1.103363, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5389.445589994501, "interactivity_tok_s": 192.6023501687139, "output_per_gpu": 1518.5235407503856, "total_per_gpu": null, "tpot_error_pct": 28.13, "tpot_ms": 5.19204464080542, "tpot_relative": 1.782189, "ttft_error_pct": 84.74, "ttft_ms": 77.98392245055592, "ttft_relative": 2.442871, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4783.163546762518, "interactivity_tok_s": 192.7600191769936, "output_per_gpu": 1501.443903796418, "total_per_gpu": 3018.871187828824, "tpot_error_pct": 28.03, "tpot_ms": 5.187797782286964, "tpot_relative": 1.780731, "ttft_error_pct": 16.48, "ttft_ms": 35.253802756250025, "ttft_relative": 1.104337, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3750.4493003332755, "interactivity_tok_s": 246.78450171162572, "output_per_gpu": 1901.1489524595147, "total_per_gpu": 3822.5362811351356, "tpot_ms": 4.0521183180641005, "tpot_relative": 1.390905, "ttft_ms": 42.21218282764312, "ttft_relative": 1.32231, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6683.631207915159, "interactivity_tok_s": 155.02103305535428, "output_per_gpu": 2448.96815680315, "total_per_gpu": null, "tpot_error_pct": 29.68, "tpot_ms": 6.4507375566444844, "tpot_relative": 2.21424, "ttft_error_pct": 57.85, "ttft_ms": 84.52668746785145, "ttft_relative": 2.647825, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6051.030466681232, "interactivity_tok_s": 153.66468090211856, "output_per_gpu": 2389.2590686420003, "total_per_gpu": 4770.960286037877, "tpot_error_pct": 30.82, "tpot_ms": 6.507676286634667, "tpot_relative": 2.233784, "ttft_error_pct": 16.96, "ttft_ms": 44.46918219375003, "ttft_relative": 1.393011, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 4644.257263662621, "interactivity_tok_s": 201.02550828388817, "output_per_gpu": 3090.9806758790673, "total_per_gpu": 6172.183771562023, "tpot_ms": 4.97449308068805, "tpot_relative": 1.707514, "ttft_ms": 53.55004263419687, "ttft_relative": 1.677472, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9104.058879121803, "interactivity_tok_s": 113.50548664425838, "output_per_gpu": 3595.758818637801, "total_per_gpu": null, "tpot_error_pct": 41.76, "tpot_ms": 8.810146800516664, "tpot_relative": 3.024116, "ttft_error_pct": 33.11, "ttft_ms": 91.2787021932558, "ttft_relative": 2.859334, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8391.88059467967, "interactivity_tok_s": 110.62586043734122, "output_per_gpu": 3437.5451622345904, "total_per_gpu": 6876.6402354177935, "tpot_error_pct": 45.45, "tpot_ms": 9.03947771386061, "tpot_relative": 3.102835, "ttft_error_pct": 8.55, "ttft_ms": 62.71092064531265, "ttft_relative": 1.96444, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 5793.488718919798, "interactivity_tok_s": 160.90909024389532, "output_per_gpu": 4937.885355892591, "total_per_gpu": 9877.997092011447, "tpot_ms": 6.214689291228148, "tpot_relative": 2.133216, "ttft_ms": 68.57579518646162, "ttft_relative": 2.148159, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12057.316504434619, "interactivity_tok_s": 85.54295044298115, "output_per_gpu": 5430.063976169136, "total_per_gpu": null, "tpot_error_pct": 46.46, "tpot_ms": 11.690034010067869, "tpot_relative": 4.012648, "ttft_error_pct": 5.8, "ttft_ms": 98.41171213518868, "ttft_relative": 3.082778, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11390.285702560926, "interactivity_tok_s": 81.4360841052465, "output_per_gpu": 5057.950762084782, "total_per_gpu": 10127.013028508904, "tpot_error_pct": 53.85, "tpot_ms": 12.279568829802995, "tpot_relative": 4.215008, "ttft_error_pct": 6.56, "ttft_ms": 97.61529882265619, "ttft_relative": 3.05783, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 7442.763835982669, "interactivity_tok_s": 125.28838154379976, "output_per_gpu": 7704.152359348615, "total_per_gpu": 15425.22950234945, "tpot_ms": 7.98158606311319, "tpot_relative": 2.73971, "ttft_ms": 104.4736775274032, "ttft_relative": 3.272671, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 72.06, "tpot_shape_error_pct": 89.02, "ttft_mape_pct": 33.43, "ttft_shape_error_pct": 41.78}, "aisimulate": {"coverage_pct": 100.0, "points": 18, "status_counts": {"failed": 0, "success": 18, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.07, "tpot_shape_error_pct": 60.58, "ttft_mape_pct": 48.29, "ttft_shape_error_pct": 18.89}, "gpu": "h100", "precisions": ["fp4"], "rows": 18, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 104.45, "tpot_shape_error_pct": 126.94, "ttft_mape_pct": 28.91, "ttft_shape_error_pct": 39.71}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 70.96, "tpot_shape_error_pct": 87.76, "ttft_mape_pct": 40.73, "ttft_shape_error_pct": 26.05}, "framework": "vllm", "id": "50ead645a1e32fa0", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 5284.959815203882, "interactivity_tok_s": 196.51029471933978, "output_per_gpu": 387.13634001795526, "total_per_gpu": null, "tpot_error_pct": 0.96, "tpot_ms": 5.088791920180169, "tpot_relative": 0.990397, "ttft_error_pct": 12.44, "ttft_ms": 79.12568085956917, "ttft_relative": 1.124392, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4654.0515081250205, "interactivity_tok_s": 198.68018771991765, "output_per_gpu": 383.8801133654787, "total_per_gpu": 771.5144276271465, "tpot_error_pct": 2.04, "tpot_ms": 5.033214491470657, "tpot_relative": 0.97958, "ttft_error_pct": 43.75, "ttft_ms": 39.587065274999745, "ttft_relative": 0.56254, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 4781.628415291198, "interactivity_tok_s": 194.62313581990577, "output_per_gpu": 373.07192763088625, "total_per_gpu": 749.7923562293687, "tpot_ms": 5.138135277633943, "tpot_relative": 1.0, "ttft_ms": 70.37199907936156, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6807.846674276907, "interactivity_tok_s": 152.2161529015276, "output_per_gpu": 601.071116284303, "total_per_gpu": null, "tpot_error_pct": 4.64, "tpot_ms": 6.569605005369731, "tpot_relative": 1.278597, "ttft_error_pct": 1.55, "ttft_ms": 87.14075378367241, "ttft_relative": 1.238287, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6107.22095193751, "interactivity_tok_s": 153.1565559767906, "output_per_gpu": 598.1620255177507, "total_per_gpu": 1191.9057996563172, "tpot_error_pct": 4.0, "tpot_ms": 6.529266694607186, "tpot_relative": 1.270746, "ttft_error_pct": 49.23, "ttft_ms": 44.938607837499966, "ttft_relative": 0.638586, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5915.742476822925, "interactivity_tok_s": 159.2828607452064, "output_per_gpu": 616.0162948880135, "total_per_gpu": 1227.4824599978401, "tpot_ms": 6.278139376210914, "tpot_relative": 1.221871, "ttft_ms": 88.51360833505169, "ttft_relative": 1.257796, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8603.819518755141, "interactivity_tok_s": 120.23212306567005, "output_per_gpu": 951.2054480175935, "total_per_gpu": null, "tpot_error_pct": 2.08, "tpot_ms": 8.31724479699827, "tpot_relative": 1.618728, "ttft_error_pct": 3.69, "ttft_ms": 95.27809142591165, "ttft_relative": 1.35392, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7664.21674878747, "interactivity_tok_s": 120.24289331418557, "output_per_gpu": 937.913111774566, "total_per_gpu": 1885.8106271328468, "tpot_error_pct": 2.07, "tpot_ms": 8.3164998149793, "tpot_relative": 1.618583, "ttft_error_pct": 46.54, "ttft_ms": 52.88045490624967, "ttft_relative": 0.751442, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 7556.882814323762, "interactivity_tok_s": 122.73016433026544, "output_per_gpu": 951.0439755051257, "total_per_gpu": 1912.2121370975294, "tpot_ms": 8.1479561724452, "tpot_relative": 1.585781, "ttft_ms": 98.92505117750261, "ttft_relative": 1.405745, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10675.658288792587, "interactivity_tok_s": 96.76597712807195, "output_per_gpu": 1533.2075603415753, "total_per_gpu": null, "tpot_error_pct": 5.68, "tpot_ms": 10.334210738930249, "tpot_relative": 2.011277, "ttft_error_pct": 9.11, "ttft_ms": 103.76070286694217, "ttft_relative": 1.47446, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9664.78591479997, "interactivity_tok_s": 96.15974796513817, "output_per_gpu": 1497.1123909782998, "total_per_gpu": 2989.4890239560355, "tpot_error_pct": 5.08, "tpot_ms": 10.399361699268812, "tpot_relative": 2.023956, "ttft_error_pct": 41.81, "ttft_ms": 66.42140231874977, "ttft_relative": 0.943861, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 10227.134469281009, "interactivity_tok_s": 91.27178082774947, "output_per_gpu": 1410.0807265592339, "total_per_gpu": 2815.7009990319975, "tpot_ms": 10.95628890913421, "tpot_relative": 2.132347, "ttft_ms": 114.15461106225848, "ttft_relative": 1.62216, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15658.453490660673, "interactivity_tok_s": 65.8081096959643, "output_per_gpu": 2090.6279167048688, "total_per_gpu": null, "tpot_error_pct": 2.44, "tpot_ms": 15.195695555153216, "tpot_relative": 2.957434, "ttft_error_pct": 47.33, "ttft_ms": 113.25693773893367, "ttft_relative": 1.609403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14449.051519218727, "interactivity_tok_s": 64.18343333474515, "output_per_gpu": 2000.5117881277229, "total_per_gpu": 4001.925561531832, "tpot_error_pct": 5.03, "tpot_ms": 15.580344460299518, "tpot_relative": 3.032295, "ttft_error_pct": 56.14, "ttft_ms": 94.30586235781217, "ttft_relative": 1.340105, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 13882.717827232773, "interactivity_tok_s": 67.41343088653466, "output_per_gpu": 2079.299971872639, "total_per_gpu": 4159.53745282218, "tpot_ms": 14.833839293584191, "tpot_relative": 2.887008, "ttft_ms": 215.02561188171967, "ttft_relative": 3.055556, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 40945.68437451774, "interactivity_tok_s": 25.07747700078899, "output_per_gpu": 6395.985413373239, "total_per_gpu": null, "tpot_error_pct": 610.9, "tpot_ms": 39.87641978371819, "tpot_relative": 7.760874, "ttft_error_pct": 99.36, "ttft_ms": 152.1069357740256, "ttft_relative": 2.16147, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51678.8676223044, "interactivity_tok_s": 35.12582929951925, "output_per_gpu": 4422.561898672376, "total_per_gpu": 8841.049825976856, "tpot_error_pct": 407.54, "tpot_ms": 28.469078736133536, "tpot_relative": 5.540741, "ttft_error_pct": 6.91, "ttft_ms": 25436.614516564186, "ttft_relative": 361.459314, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19948006055", "measured": {"e2e_ms": 28935.737961810082, "interactivity_tok_s": 178.27605186215928, "output_per_gpu": 343.05833083025374, "total_per_gpu": 689.4716411142671, "tpot_ms": 5.6092783610284735, "tpot_relative": 1.091695, "ttft_ms": 23793.330964213237, "ttft_relative": 338.10793, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 71.71, "tpot_shape_error_pct": 92.27, "ttft_mape_pct": 32.95, "ttft_shape_error_pct": 41.59}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 46.85, "tpot_shape_error_pct": 62.3, "ttft_mape_pct": 47.95, "ttft_shape_error_pct": 17.98}, "framework": "vllm", "id": "c58113d6e58a7088", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 3937.6210315528588, "interactivity_tok_s": 264.0568783288069, "output_per_gpu": 259.8015379851232, "total_per_gpu": null, "tpot_error_pct": 7.54, "tpot_ms": 3.787062871942263, "tpot_relative": 0.924603, "ttft_error_pct": 27.01, "ttft_ms": 63.45571355592405, "ttft_relative": 1.270066, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3474.9896196250047, "interactivity_tok_s": 265.8110500343236, "output_per_gpu": 256.13114400061613, "total_per_gpu": 514.7671527672629, "tpot_error_pct": 8.15, "tpot_ms": 3.7620708389319106, "tpot_relative": 0.918501, "ttft_error_pct": 48.01, "ttft_ms": 25.973655024999836, "ttft_relative": 0.519863, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 3805.091871647164, "interactivity_tok_s": 244.14767034431566, "output_per_gpu": 233.89616537930232, "total_per_gpu": 470.0797459257617, "tpot_ms": 4.095881802147543, "tpot_relative": 1.0, "ttft_ms": 49.96251328266226, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4847.2400350242915, "interactivity_tok_s": 214.13395386635386, "output_per_gpu": 422.09587006552, "total_per_gpu": null, "tpot_error_pct": 2.47, "tpot_ms": 4.669974013668678, "tpot_relative": 1.140163, "ttft_error_pct": 17.39, "ttft_ms": 69.8566190412341, "ttft_relative": 1.398181, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4344.800990325004, "interactivity_tok_s": 215.13799155116158, "output_per_gpu": 419.52995659784887, "total_per_gpu": 835.9611059657617, "tpot_error_pct": 2.92, "tpot_ms": 4.648179490706976, "tpot_relative": 1.134842, "ttft_error_pct": 50.62, "ttft_ms": 29.388079200000096, "ttft_relative": 0.588203, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 4503.93655274238, "interactivity_tok_s": 208.84608263995622, "output_per_gpu": 403.8485923095754, "total_per_gpu": 804.7142058879133, "tpot_ms": 4.788215260536953, "tpot_relative": 1.169032, "ttft_ms": 59.50904918718152, "ttft_relative": 1.191074, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5811.8226917147485, "interactivity_tok_s": 178.36325743686984, "output_per_gpu": 704.0820439745171, "total_per_gpu": null, "tpot_error_pct": 4.31, "tpot_ms": 5.606535866020172, "tpot_relative": 1.368823, "ttft_error_pct": 3.42, "ttft_ms": 76.33650077611175, "ttft_relative": 1.527876, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5174.189802825016, "interactivity_tok_s": 178.05889762647135, "output_per_gpu": 693.6275466711406, "total_per_gpu": 1394.6389941277637, "tpot_error_pct": 4.15, "tpot_ms": 5.6161192354328815, "tpot_relative": 1.371162, "ttft_error_pct": 53.3, "ttft_ms": 34.46511185624977, "ttft_relative": 0.689819, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5436.777537528542, "interactivity_tok_s": 170.67749098023017, "output_per_gpu": 659.8343346273936, "total_per_gpu": 1326.6928298221205, "tpot_ms": 5.859003400254059, "tpot_relative": 1.430462, "ttft_ms": 73.80867890897207, "ttft_relative": 1.477281, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7253.166228249206, "interactivity_tok_s": 142.67395396566152, "output_per_gpu": 1128.3348185410996, "total_per_gpu": null, "tpot_error_pct": 5.46, "tpot_ms": 7.008987780913943, "tpot_relative": 1.711228, "ttft_error_pct": 16.39, "ttft_ms": 82.97172837424277, "ttft_relative": 1.66068, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6583.667694328085, "interactivity_tok_s": 141.12227851695457, "output_per_gpu": 1097.4559116776372, "total_per_gpu": 2191.440283312381, "tpot_error_pct": 4.42, "tpot_ms": 7.086053389365159, "tpot_relative": 1.730043, "ttft_error_pct": 56.1, "ttft_ms": 43.56828233750022, "ttft_relative": 0.872019, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 6941.638542457076, "interactivity_tok_s": 134.88902335998654, "output_per_gpu": 1034.0356959599537, "total_per_gpu": 2064.800466604266, "tpot_ms": 7.413501670415681, "tpot_relative": 1.809989, "ttft_ms": 99.24114244495286, "ttft_relative": 1.986312, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9594.356622109572, "interactivity_tok_s": 107.63589600155531, "output_per_gpu": 1706.0028769705107, "total_per_gpu": null, "tpot_error_pct": 4.7, "tpot_ms": 9.290580904213872, "tpot_relative": 2.268274, "ttft_error_pct": 34.1, "ttft_ms": 90.092357098779, "ttft_relative": 1.803199, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8838.03417219378, "interactivity_tok_s": 104.97857875137343, "output_per_gpu": 1629.56916723791, "total_per_gpu": 3259.8730701591708, "tpot_error_pct": 2.29, "tpot_ms": 9.525752890676442, "tpot_relative": 2.32569, "ttft_error_pct": 55.22, "ttft_ms": 61.22187691875022, "ttft_relative": 1.225356, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 9118.677798075078, "interactivity_tok_s": 102.57294373409343, "output_per_gpu": 1576.7853870125666, "total_per_gpu": 3154.2817107022106, "tpot_ms": 9.749159608720655, "tpot_relative": 2.380235, "ttft_ms": 136.70222277787616, "ttft_relative": 2.736096, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24416.895731445195, "interactivity_tok_s": 42.097990445419555, "output_per_gpu": 5362.84388647179, "total_per_gpu": null, "tpot_error_pct": 405.79, "tpot_ms": 23.75410297307444, "tpot_relative": 5.799509, "ttft_error_pct": 99.42, "ttft_ms": 116.44838999004298, "ttft_relative": 2.330715, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 30640.83828065291, "interactivity_tok_s": 59.287686979447294, "output_per_gpu": 3728.12866888163, "total_per_gpu": 7452.823063738193, "tpot_error_pct": 259.15, "tpot_ms": 16.866908644060622, "tpot_relative": 4.118016, "ttft_error_pct": 24.45, "ttft_ms": 15092.430457504819, "ttft_relative": 302.075085, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19948006055", "measured": {"e2e_ms": 24283.01082472317, "interactivity_tok_s": 212.92881725946063, "output_per_gpu": 204.9376026665753, "total_per_gpu": 411.8794168169093, "tpot_ms": 4.696405178362813, "tpot_relative": 1.146616, "ttft_ms": 19976.768855191767, "ttft_relative": 399.835147, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.02, "tpot_shape_error_pct": 47.85, "ttft_mape_pct": 38.42, "ttft_shape_error_pct": 44.06}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.39, "tpot_shape_error_pct": 31.68, "ttft_mape_pct": 56.18, "ttft_shape_error_pct": 12.63}, "framework": "vllm", "id": "b5a22fb43eb96bf7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 3550.5631045954347, "interactivity_tok_s": 292.7000783768679, "output_per_gpu": 144.0616558364993, "total_per_gpu": null, "tpot_error_pct": 8.88, "tpot_ms": 3.416466457902493, "tpot_relative": 0.911152, "ttft_error_pct": 41.91, "ttft_ms": 55.517918161184454, "ttft_relative": 1.419061, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3140.0714648250023, "interactivity_tok_s": 293.75181368171053, "output_per_gpu": 141.5051752083568, "total_per_gpu": 284.39421698621425, "tpot_error_pct": 9.21, "tpot_ms": 3.4042343006042914, "tpot_relative": 0.90789, "ttft_error_pct": 51.03, "ttft_ms": 19.159893725000003, "ttft_relative": 0.489735, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 3476.492109033279, "interactivity_tok_s": 266.69436447511947, "output_per_gpu": 126.85038401547207, "total_per_gpu": 254.94131633957568, "tpot_ms": 3.749610540020587, "tpot_relative": 1.0, "ttft_ms": 39.12299524527043, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3886.823788582785, "interactivity_tok_s": 267.40102728123503, "output_per_gpu": 263.19690720350525, "total_per_gpu": null, "tpot_error_pct": 7.91, "tpot_ms": 3.7397014146406584, "tpot_relative": 0.997357, "ttft_error_pct": 28.73, "ttft_ms": 61.10924140539146, "ttft_relative": 1.561978, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3489.8106888125003, "interactivity_tok_s": 267.6500328640341, "output_per_gpu": 260.38755996814695, "total_per_gpu": 518.8517987509414, "tpot_error_pct": 7.99, "tpot_ms": 3.7362222201108373, "tpot_relative": 0.996429, "ttft_error_pct": 54.75, "ttft_ms": 21.48116171250026, "ttft_relative": 0.549067, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 3816.39217142656, "interactivity_tok_s": 246.26131080620408, "output_per_gpu": 237.95655343179038, "total_per_gpu": 474.1554695921811, "tpot_ms": 4.060727187418215, "tpot_relative": 1.082973, "ttft_ms": 47.47176341188606, "ttft_relative": 1.213398, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4713.777809959265, "interactivity_tok_s": 220.14074796522513, "output_per_gpu": 434.0467630182342, "total_per_gpu": null, "tpot_error_pct": 6.2, "tpot_ms": 4.542548388896937, "tpot_relative": 1.211472, "ttft_error_pct": 14.05, "ttft_ms": 66.75080811769872, "ttft_relative": 1.706178, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4193.6841483625185, "interactivity_tok_s": 219.54329685963248, "output_per_gpu": 427.56168748374097, "total_per_gpu": 859.6749143277697, "tpot_error_pct": 5.95, "tpot_ms": 4.554910189944726, "tpot_relative": 1.214769, "ttft_error_pct": 56.83, "ttft_ms": 25.264308093750174, "ttft_relative": 0.645766, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 4490.54153215875, "interactivity_tok_s": 206.48453504822885, "output_per_gpu": 398.84960211787467, "total_per_gpu": 801.9450936968897, "tpot_ms": 4.842977706618216, "tpot_relative": 1.291595, "ttft_ms": 58.526536234421656, "ttft_relative": 1.495963, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5434.884955550698, "interactivity_tok_s": 190.77383042163467, "output_per_gpu": 752.9138212614423, "total_per_gpu": null, "tpot_error_pct": 10.07, "tpot_ms": 5.241809098186431, "tpot_relative": 1.397961, "ttft_error_pct": 23.03, "ttft_ms": 72.51424810597854, "ttft_relative": 1.853494, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4934.2618573375, "interactivity_tok_s": 188.26320880317203, "output_per_gpu": 730.020915672692, "total_per_gpu": 1457.7325842822984, "tpot_error_pct": 8.87, "tpot_ms": 5.311712290240912, "tpot_relative": 1.416604, "ttft_error_pct": 66.25, "ttft_ms": 31.793135662499907, "ttft_relative": 0.812646, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5473.561997280922, "interactivity_tok_s": 171.56478653219878, "output_per_gpu": 657.6515979442268, "total_per_gpu": 1313.2228719025484, "tpot_ms": 5.828701916126144, "tpot_relative": 1.554482, "ttft_ms": 94.2055951396469, "ttft_relative": 2.407934, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7188.122765776361, "interactivity_tok_s": 143.88991621898575, "output_per_gpu": 1138.5448282777177, "total_per_gpu": null, "tpot_error_pct": 5.84, "tpot_ms": 6.949757330305913, "tpot_relative": 1.853461, "ttft_error_pct": 23.25, "ttft_ms": 78.52101687341191, "ttft_relative": 2.00703, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6649.832911049993, "interactivity_tok_s": 139.49517273015434, "output_per_gpu": 1079.688702844311, "total_per_gpu": 2159.8642127741014, "tpot_error_pct": 2.87, "tpot_ms": 7.168706847902504, "tpot_relative": 1.911854, "ttft_error_pct": 56.05, "ttft_ms": 44.964175367187416, "ttft_relative": 1.149303, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 6901.786254979379, "interactivity_tok_s": 135.49043079592022, "output_per_gpu": 1038.3787555592578, "total_per_gpu": 2077.225692488096, "tpot_ms": 7.3805950289303475, "tpot_relative": 1.968363, "ttft_ms": 102.31283150096715, "ttft_relative": 2.615158, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16531.95582362037, "interactivity_tok_s": 62.250677014146845, "output_per_gpu": 3960.329963285744, "total_per_gpu": null, "tpot_error_pct": 201.24, "tpot_ms": 16.064082319502226, "tpot_relative": 4.2842, "ttft_error_pct": 99.56, "ttft_ms": 98.39961076959374, "ttft_relative": 2.515135, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21650.02178675271, "interactivity_tok_s": 83.92096551479608, "output_per_gpu": 2636.0246672150283, "total_per_gpu": 5269.621083731626, "tpot_error_pct": 123.45, "tpot_ms": 11.915973486073517, "tpot_relative": 3.177923, "ttft_error_pct": 52.16, "ttft_ms": 10665.436010790309, "ttft_relative": 272.612972, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19948006055", "measured": {"e2e_ms": 27174.871303641703, "interactivity_tok_s": 187.52156175606393, "output_per_gpu": 362.22519771742236, "total_per_gpu": 728.3064056737451, "tpot_ms": 5.332720091681205, "tpot_relative": 1.422206, "ttft_ms": 22294.927658548113, "ttft_relative": 569.867606, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 36, "status_counts": {"failed": 0, "success": 36, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.9, "tpot_shape_error_pct": 5.4, "ttft_mape_pct": 48.36, "ttft_shape_error_pct": 35.47}, "aisimulate": {"coverage_pct": 100.0, "points": 36, "status_counts": {"failed": 0, "success": 36, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.29, "tpot_shape_error_pct": 5.34, "ttft_mape_pct": 65.13, "ttft_shape_error_pct": 33.57}, "gpu": "h200", "precisions": ["fp4"], "rows": 36, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.48, "tpot_shape_error_pct": 6.94, "ttft_mape_pct": 71.16, "ttft_shape_error_pct": 18.75}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.29, "tpot_shape_error_pct": 7.78, "ttft_mape_pct": 80.56, "ttft_shape_error_pct": 34.42}, "framework": "trt", "id": "509e794fd7ed4bfc", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 6878.9423235144695, "interactivity_tok_s": 151.01383283458696, "output_per_gpu": 591.822847537918, "total_per_gpu": null, "tpot_error_pct": 11.4, "tpot_ms": 6.62190993520011, "tpot_relative": 0.885987, "ttft_error_pct": 74.92, "ttft_ms": 104.7284598047572, "ttft_relative": 0.250765, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6173.2643066249875, "interactivity_tok_s": 150.0611766043566, "output_per_gpu": 579.2088248820423, "total_per_gpu": 1164.0820908584392, "tpot_error_pct": 10.84, "tpot_ms": 6.663948814932641, "tpot_relative": 0.891611, "ttft_error_pct": 84.76, "ttft_ms": 63.662651624999725, "ttft_relative": 0.152436, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 7276.110471365973, "interactivity_tok_s": 133.7962606841947, "output_per_gpu": 492.5029747691003, "total_per_gpu": 989.8224405333818, "tpot_ms": 7.47405043224896, "tpot_relative": 1.0, "ttft_ms": 417.6355976611376, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9922.947453940638, "interactivity_tok_s": 104.31170175208844, "output_per_gpu": 823.1433704729392, "total_per_gpu": null, "tpot_error_pct": 3.46, "tpot_ms": 9.586652151228842, "tpot_relative": 1.282658, "ttft_error_pct": 64.74, "ttft_ms": 115.80230323353337, "ttft_relative": 0.277281, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9000.598360799793, "interactivity_tok_s": 104.02315680969397, "output_per_gpu": 813.023289948425, "total_per_gpu": 1620.041281802217, "tpot_error_pct": 3.19, "tpot_ms": 9.613244114764353, "tpot_relative": 1.286216, "ttft_error_pct": 77.39, "ttft_ms": 74.28009517500041, "ttft_relative": 0.177859, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 9552.353765419684, "interactivity_tok_s": 100.70448921379653, "output_per_gpu": 765.7647793571812, "total_per_gpu": 1525.8733298864004, "tpot_ms": 9.930043911716696, "tpot_relative": 1.328603, "ttft_ms": 328.4683411824517, "ttft_relative": 0.786495, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13292.91924191372, "interactivity_tok_s": 77.76618885031458, "output_per_gpu": 1232.0253413072828, "total_per_gpu": null, "tpot_error_pct": 2.49, "tpot_ms": 12.859058863291008, "tpot_relative": 1.720494, "ttft_error_pct": 69.2, "ttft_ms": 138.10202476701883, "ttft_relative": 0.330676, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11942.004851681213, "interactivity_tok_s": 77.24942507548647, "output_per_gpu": 1205.4519429051809, "total_per_gpu": 2423.736331105818, "tpot_error_pct": 1.84, "tpot_ms": 12.945080161086267, "tpot_relative": 1.732003, "ttft_error_pct": 79.04, "ttft_ms": 93.9952683187494, "ttft_relative": 0.225065, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 12520.153532776749, "interactivity_tok_s": 75.83056883365659, "output_per_gpu": 1151.2086314730122, "total_per_gpu": 2314.672269770628, "tpot_ms": 13.187293928832572, "tpot_relative": 1.764411, "ttft_ms": 448.35579777136445, "ttft_relative": 1.073557, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16241.054697994372, "interactivity_tok_s": 63.702707859496755, "output_per_gpu": 2022.499948966317, "total_per_gpu": null, "tpot_error_pct": 10.23, "tpot_ms": 15.69791981536497, "tpot_relative": 2.100323, "ttft_error_pct": 72.22, "ttft_ms": 182.082726876008, "ttft_relative": 0.435985, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14840.251281599976, "interactivity_tok_s": 62.70155503885323, "output_per_gpu": 1951.8891031868054, "total_per_gpu": 3897.603870704269, "tpot_error_pct": 8.8, "tpot_ms": 15.94856777284625, "tpot_relative": 2.133859, "ttft_error_pct": 81.7, "ttft_ms": 119.89791656562461, "ttft_relative": 0.287087, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 16812.3635148193, "interactivity_tok_s": 57.18412595579454, "output_per_gpu": 1726.484361103201, "total_per_gpu": 3447.50740068157, "tpot_ms": 17.487370547082197, "tpot_relative": 2.339745, "ttft_ms": 655.3529687953414, "ttft_relative": 1.569198, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23493.67851506556, "interactivity_tok_s": 43.94744045174448, "output_per_gpu": 2798.642216886642, "total_per_gpu": null, "tpot_error_pct": 4.83, "tpot_ms": 22.75445372291995, "tpot_relative": 3.044461, "ttft_error_pct": 74.73, "ttft_ms": 215.87235651844952, "ttft_relative": 0.516892, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21807.68091809382, "interactivity_tok_s": 42.58312236880192, "output_per_gpu": 2652.735324463848, "total_per_gpu": 5306.666706965952, "tpot_error_pct": 1.79, "tpot_ms": 23.483482289984437, "tpot_relative": 3.142002, "ttft_error_pct": 79.91, "ttft_ms": 171.55656901874943, "ttft_relative": 0.410781, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 22939.92579965561, "interactivity_tok_s": 41.82260223720729, "output_per_gpu": 2524.323669114356, "total_per_gpu": 5049.785498371475, "tpot_ms": 23.91051600109078, "tpot_relative": 3.199138, "ttft_ms": 854.1142078014673, "ttft_relative": 2.045118, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.43, "tpot_shape_error_pct": 10.75, "ttft_mape_pct": 86.79, "ttft_shape_error_pct": 7.81}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.7, "tpot_shape_error_pct": 10.87, "ttft_mape_pct": 92.03, "ttft_shape_error_pct": 5.21}, "framework": "trt", "id": "35ba5b0522d46b35", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 4977.755500473214, "interactivity_tok_s": 207.5846999005982, "output_per_gpu": 102.48184673943403, "total_per_gpu": null, "tpot_error_pct": 18.02, "tpot_ms": 4.817310719329745, "tpot_relative": 0.819819, "ttft_error_pct": 86.25, "ttft_ms": 49.64663459888442, "ttft_relative": 0.137522, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4490.209502624998, "interactivity_tok_s": 205.52526595198185, "output_per_gpu": 99.13028926751069, "total_per_gpu": 199.23003490395098, "tpot_error_pct": 17.2, "tpot_ms": 4.865581831839771, "tpot_relative": 0.828034, "ttft_error_pct": 91.82, "ttft_ms": 29.545166099999697, "ttft_relative": 0.081841, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 5756.239419290796, "interactivity_tok_s": 170.18192037768344, "output_per_gpu": 77.63284262705243, "total_per_gpu": 156.02490480525339, "tpot_ms": 5.876064847433309, "tpot_relative": 1.0, "ttft_ms": 361.0080464510247, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5506.065027366148, "interactivity_tok_s": 187.64946838586036, "output_per_gpu": 185.5430977066562, "total_per_gpu": null, "tpot_error_pct": 26.83, "tpot_ms": 5.3290851746178, "tpot_relative": 0.906914, "ttft_error_pct": 87.32, "ttft_ms": 54.41089373213756, "ttft_relative": 0.150719, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5023.683923425013, "interactivity_tok_s": 186.02416681902096, "output_per_gpu": 181.0564942320104, "total_per_gpu": 360.7756365907413, "tpot_error_pct": 26.2, "tpot_ms": 5.375645633036911, "tpot_relative": 0.914838, "ttft_error_pct": 92.24, "ttft_ms": 33.29433655000008, "ttft_relative": 0.092226, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 7197.14968316257, "interactivity_tok_s": 137.29500024121327, "output_per_gpu": 126.95039143064665, "total_per_gpu": 252.96307916547454, "tpot_ms": 7.283586425165536, "tpot_relative": 1.239535, "ttft_ms": 429.186265647877, "ttft_relative": 1.188855, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.99, "tpot_shape_error_pct": 13.07, "ttft_mape_pct": 68.01, "ttft_shape_error_pct": 91.63}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.19, "tpot_shape_error_pct": 12.69, "ttft_mape_pct": 81.13, "ttft_shape_error_pct": 95.41}, "framework": "trt", "id": "960d465d25342020", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5350.284804592734, "interactivity_tok_s": 193.648299290249, "output_per_gpu": 190.4587154868776, "total_per_gpu": null, "tpot_error_pct": 17.34, "tpot_ms": 5.164000942250228, "tpot_relative": 0.826617, "ttft_error_pct": 81.04, "ttft_ms": 67.51184067075104, "ttft_relative": 0.1896, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4813.660149199975, "interactivity_tok_s": 192.01984485845358, "output_per_gpu": 185.13704905183155, "total_per_gpu": 372.08466773535037, "tpot_error_pct": 16.64, "tpot_ms": 5.207795062729817, "tpot_relative": 0.833627, "ttft_error_pct": 89.0, "ttft_ms": 39.17078830000033, "ttft_relative": 0.110007, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 6091.608534310944, "interactivity_tok_s": 160.07301582086893, "output_per_gpu": 146.85195599660628, "total_per_gpu": 295.14006803676546, "tpot_ms": 6.2471491204929785, "tpot_relative": 1.0, "ttft_ms": 356.07514197472483, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6407.024078772253, "interactivity_tok_s": 161.54011811394906, "output_per_gpu": 318.72201070643575, "total_per_gpu": null, "tpot_error_pct": 8.44, "tpot_ms": 6.190412707848884, "tpot_relative": 0.990918, "ttft_error_pct": 44.6, "ttft_ms": 74.2318786428453, "ttft_relative": 0.208473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5827.076672862517, "interactivity_tok_s": 160.54657368179323, "output_per_gpu": 312.7454889142715, "total_per_gpu": 623.1809211402518, "tpot_error_pct": 7.88, "tpot_ms": 6.228722152501501, "tpot_relative": 0.99705, "ttft_error_pct": 67.01, "ttft_ms": 44.207929774999684, "ttft_relative": 0.124153, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 6412.396784487646, "interactivity_tok_s": 147.9023782412324, "output_per_gpu": 284.0584375311626, "total_per_gpu": 566.0187118057985, "tpot_ms": 6.761216498959709, "tpot_relative": 1.082288, "ttft_ms": 133.98434043629095, "ttft_relative": 0.376281, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7227.429132199247, "interactivity_tok_s": 143.29080350701162, "output_per_gpu": 566.3583518537894, "total_per_gpu": null, "tpot_error_pct": 25.26, "tpot_ms": 6.978814938050559, "tpot_relative": 1.11712, "ttft_error_pct": 73.31, "ttft_ms": 88.1014505735249, "ttft_relative": 0.247424, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6519.909067662493, "interactivity_tok_s": 141.5386657968201, "output_per_gpu": 549.7943011893055, "total_per_gpu": 1105.4413494211537, "tpot_error_pct": 24.34, "tpot_ms": 7.065207195293956, "tpot_relative": 1.130949, "ttft_error_pct": 83.72, "ttft_ms": 53.75772930624972, "ttft_relative": 0.150973, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 8877.566549746552, "interactivity_tok_s": 107.09450219805514, "output_per_gpu": 405.7143353353321, "total_per_gpu": 815.7476375481209, "tpot_ms": 9.337547488204864, "tpot_relative": 1.494689, "ttft_ms": 330.1494436105713, "ttft_relative": 0.92719, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8470.325245611593, "interactivity_tok_s": 122.45701359598178, "output_per_gpu": 969.8045418647282, "total_per_gpu": null, "tpot_error_pct": 32.93, "tpot_ms": 8.166130878377173, "tpot_relative": 1.307177, "ttft_error_pct": 73.09, "ttft_ms": 116.37335703174566, "ttft_relative": 0.326822, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7718.598158453137, "interactivity_tok_s": 120.6243885118805, "output_per_gpu": 933.7894157470529, "total_per_gpu": 1864.6250113780513, "tpot_error_pct": 31.92, "tpot_ms": 8.290197466174167, "tpot_relative": 1.327037, "ttft_error_pct": 84.79, "ttft_ms": 65.77888328437488, "ttft_relative": 0.184733, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 11689.868731540628, "interactivity_tok_s": 82.12652467781568, "output_per_gpu": 620.7192858119649, "total_per_gpu": 1239.4750741994194, "tpot_ms": 12.176334064092252, "tpot_relative": 1.949103, "ttft_ms": 432.4044505803613, "ttft_relative": 1.214363, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.78, "tpot_shape_error_pct": 4.18, "ttft_mape_pct": 81.06, "ttft_shape_error_pct": 43.63}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.93, "tpot_shape_error_pct": 3.32, "ttft_mape_pct": 86.98, "ttft_shape_error_pct": 65.23}, "framework": "trt", "id": "7d45677e0c15f27f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 5856.077855646596, "interactivity_tok_s": 176.7698767517486, "output_per_gpu": 348.15519186793705, "total_per_gpu": null, "tpot_error_pct": 16.21, "tpot_ms": 5.657072451345181, "tpot_relative": 0.83793, "ttft_error_pct": 85.96, "ttft_ms": 68.89273792047545, "ttft_relative": 0.140368, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5243.1419126250075, "interactivity_tok_s": 176.26731522847848, "output_per_gpu": 340.32990019507486, "total_per_gpu": 683.9880968343468, "tpot_error_pct": 15.97, "tpot_ms": 5.673201516139255, "tpot_relative": 0.840319, "ttft_error_pct": 91.44, "ttft_ms": 41.99289150000009, "ttft_relative": 0.08556, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 6687.223871797323, "interactivity_tok_s": 148.1207375579592, "output_per_gpu": 267.5957145035627, "total_per_gpu": 537.8084128940953, "tpot_ms": 6.751249126130654, "tpot_relative": 1.0, "ttft_ms": 490.79990547616035, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7560.10459314438, "interactivity_tok_s": 136.69710114216025, "output_per_gpu": 540.4875544299248, "total_per_gpu": null, "tpot_error_pct": 13.12, "tpot_ms": 7.31544408509464, "tpot_relative": 1.083569, "ttft_error_pct": 85.03, "ttft_ms": 76.40529409256348, "ttft_relative": 0.155675, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6867.91994816253, "interactivity_tok_s": 136.16937793510306, "output_per_gpu": 531.5765323723948, "total_per_gpu": 1059.2266390489033, "tpot_error_pct": 12.78, "tpot_ms": 7.3437950232583855, "tpot_relative": 1.087768, "ttft_error_pct": 90.28, "ttft_ms": 49.608163887499884, "ttft_relative": 0.101076, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 8332.343178860538, "interactivity_tok_s": 118.76301988888262, "output_per_gpu": 438.65203989164957, "total_per_gpu": 874.0640296003145, "tpot_ms": 8.420129438739623, "tpot_relative": 1.247196, "ttft_ms": 510.46166033738706, "ttft_relative": 1.040061, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9175.840794593843, "interactivity_tok_s": 112.61557551446126, "output_per_gpu": 892.4549236966325, "total_per_gpu": null, "tpot_error_pct": 17.15, "tpot_ms": 8.87976636829945, "tpot_relative": 1.315278, "ttft_error_pct": 76.22, "ttft_ms": 91.83979982350611, "ttft_relative": 0.187123, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8237.804757162494, "interactivity_tok_s": 111.95628262449735, "output_per_gpu": 871.7318761582454, "total_per_gpu": 1752.743634172375, "tpot_error_pct": 16.66, "tpot_ms": 8.932057911872722, "tpot_relative": 1.323023, "ttft_error_pct": 83.64, "ttft_ms": 63.15946468125012, "ttft_relative": 0.128687, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 10201.772361877374, "interactivity_tok_s": 93.30104224489149, "output_per_gpu": 706.223103782214, "total_per_gpu": 1419.9641923327229, "tpot_ms": 10.717993882375445, "tpot_relative": 1.587557, "ttft_ms": 386.1593430337962, "ttft_relative": 0.786796, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11617.105405706969, "interactivity_tok_s": 89.00979505605585, "output_per_gpu": 1413.6590645941287, "total_per_gpu": null, "tpot_error_pct": 22.08, "tpot_ms": 11.234718598893844, "tpot_relative": 1.664095, "ttft_error_pct": 79.55, "ttft_ms": 123.98827903856663, "ttft_relative": 0.252625, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10589.491029362563, "interactivity_tok_s": 87.84514341902374, "output_per_gpu": 1365.671804165972, "total_per_gpu": 2727.023631280323, "tpot_error_pct": 21.05, "tpot_ms": 11.383668590874427, "tpot_relative": 1.686157, "ttft_error_pct": 86.36, "ttft_ms": 82.67161810937543, "ttft_relative": 0.168443, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 13936.826969214599, "interactivity_tok_s": 69.35429699950478, "output_per_gpu": 1041.806557785582, "total_per_gpu": 2080.3176089874205, "tpot_ms": 14.418717271507207, "tpot_relative": 2.135711, "ttft_ms": 606.1791965999873, "ttft_relative": 1.235084, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15563.114423033363, "interactivity_tok_s": 66.378589323727, "output_per_gpu": 2112.6672466355535, "total_per_gpu": null, "tpot_error_pct": 20.32, "tpot_ms": 15.065098704086958, "tpot_relative": 2.231454, "ttft_error_pct": 78.56, "ttft_ms": 151.518448752404, "ttft_relative": 0.308717, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14373.024779821717, "interactivity_tok_s": 64.63984836342512, "output_per_gpu": 2006.351519637406, "total_per_gpu": 4013.6076575533484, "tpot_error_pct": 18.18, "tpot_ms": 15.470333320983247, "tpot_relative": 2.291477, "ttft_error_pct": 83.16, "ttft_ms": 118.97088830937525, "ttft_relative": 0.242402, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 18186.04809330718, "interactivity_tok_s": 52.89038375469444, "output_per_gpu": 1592.373396844084, "total_per_gpu": 3185.4647586444285, "tpot_ms": 18.907028631858662, "tpot_relative": 2.800523, "ttft_ms": 706.6360079450533, "ttft_relative": 1.439764, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.53, "tpot_shape_error_pct": 4.99, "ttft_mape_pct": 18.5, "ttft_shape_error_pct": 23.3}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.06, "tpot_shape_error_pct": 3.41, "ttft_mape_pct": 38.81, "ttft_shape_error_pct": 8.9}, "framework": "vllm", "id": "711298c3dd479aeb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 6647.349221004139, "interactivity_tok_s": 156.3916699140724, "output_per_gpu": 615.5837257760122, "total_per_gpu": null, "tpot_error_pct": 0.83, "tpot_ms": 6.394202456879183, "tpot_relative": 1.008265, "ttft_error_pct": 10.1, "ttft_ms": 106.08010761673444, "ttft_relative": 1.100992, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5843.801523949999, "interactivity_tok_s": 158.5899095925532, "output_per_gpu": 612.7940084501637, "total_per_gpu": 1231.5809082630344, "tpot_error_pct": 0.57, "tpot_ms": 6.305571411000768, "tpot_relative": 0.994289, "ttft_error_pct": 34.93, "ttft_ms": 62.691696524999735, "ttft_relative": 0.650669, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5909.657886461355, "interactivity_tok_s": 157.68426880650594, "output_per_gpu": 604.1920033325681, "total_per_gpu": 1214.2927737030939, "tpot_ms": 6.341786708140798, "tpot_relative": 1.0, "ttft_ms": 96.34958568494767, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9355.249926490216, "interactivity_tok_s": 110.73309894383763, "output_per_gpu": 874.802925021413, "total_per_gpu": null, "tpot_error_pct": 0.31, "tpot_ms": 9.03072351029557, "tpot_relative": 1.424003, "ttft_error_pct": 10.35, "ttft_ms": 116.81977545784815, "ttft_relative": 1.212457, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8380.08680050002, "interactivity_tok_s": 111.75232208638585, "output_per_gpu": 873.5041062531259, "total_per_gpu": 1740.5561801846825, "tpot_error_pct": 1.22, "tpot_ms": 8.9483599206734, "tpot_relative": 1.411016, "ttft_error_pct": 33.04, "ttft_ms": 70.88950005000031, "ttft_relative": 0.735753, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 8515.834093489684, "interactivity_tok_s": 110.38529248524974, "output_per_gpu": 857.9473332912776, "total_per_gpu": 1709.5575424809365, "tpot_ms": 9.059177880365043, "tpot_relative": 1.42849, "ttft_ms": 105.86689785122871, "ttft_relative": 1.098779, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12593.40212296967, "interactivity_tok_s": 82.06541368625805, "output_per_gpu": 1299.728210071659, "total_per_gpu": null, "tpot_error_pct": 3.68, "tpot_ms": 12.185401316845505, "tpot_relative": 1.921446, "ttft_error_pct": 4.13, "ttft_ms": 127.73657583671724, "ttft_relative": 1.325762, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11199.953472187466, "interactivity_tok_s": 82.33306180554192, "output_per_gpu": 1285.7281350596434, "total_per_gpu": 2585.143282741473, "tpot_error_pct": 3.99, "tpot_ms": 12.145789043553936, "tpot_relative": 1.9152, "ttft_error_pct": 37.46, "ttft_ms": 83.3362399249994, "ttft_relative": 0.864936, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 11714.082566532306, "interactivity_tok_s": 79.04398106418866, "output_per_gpu": 1228.7675597214197, "total_per_gpu": 2470.615767397129, "tpot_ms": 12.651184651086052, "tpot_relative": 1.994893, "ttft_ms": 133.24289215379395, "ttft_relative": 1.382911, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16356.236400915635, "interactivity_tok_s": 63.08221482595864, "output_per_gpu": 2001.4384237053098, "total_per_gpu": null, "tpot_error_pct": 10.12, "tpot_ms": 15.85232862794943, "tpot_relative": 2.499663, "ttft_error_pct": 23.73, "ttft_ms": 139.30421452336896, "ttft_relative": 1.445821, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14835.656475640691, "interactivity_tok_s": 62.6551735450099, "output_per_gpu": 1955.3395966730575, "total_per_gpu": 3904.4939428635425, "tpot_error_pct": 9.51, "tpot_ms": 15.960373955099257, "tpot_relative": 2.5167, "ttft_error_pct": 42.8, "ttft_ms": 104.4737342843752, "ttft_relative": 1.084319, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 16462.77201730336, "interactivity_tok_s": 56.696996540700965, "output_per_gpu": 1763.6305906137486, "total_per_gpu": 3521.682356464661, "tpot_ms": 17.637618586764678, "tpot_relative": 2.781175, "ttft_ms": 182.63708927079279, "ttft_relative": 1.895567, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24796.212047509536, "interactivity_tok_s": 41.511055683810234, "output_per_gpu": 2640.4032952515354, "total_per_gpu": null, "tpot_error_pct": 2.71, "tpot_ms": 24.08996792606291, "tpot_relative": 3.798609, "ttft_error_pct": 44.2, "ttft_ms": 152.17485914717707, "ttft_relative": 1.579403, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22955.842605209276, "interactivity_tok_s": 40.39526512426553, "output_per_gpu": 2524.851090728019, "total_per_gpu": 5050.840579401212, "tpot_error_pct": 0.02, "tpot_ms": 24.755376575045617, "tpot_relative": 3.903533, "ttft_error_pct": 45.81, "ttft_ms": 147.76627073749918, "ttft_relative": 1.533647, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019407598", "measured": {"e2e_ms": 23086.35838260525, "interactivity_tok_s": 40.385962026217186, "output_per_gpu": 2508.8588618150784, "total_per_gpu": 5018.848911042956, "tpot_ms": 24.7610790935433, "tpot_relative": 3.904433, "ttft_ms": 272.70323254342657, "ttft_relative": 2.830352, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.63, "tpot_shape_error_pct": 1.93, "ttft_mape_pct": 27.63, "ttft_shape_error_pct": 20.66}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.88, "tpot_shape_error_pct": 1.77, "ttft_mape_pct": 54.27, "ttft_shape_error_pct": 9.34}, "framework": "vllm", "id": "be34520b48ec4a96", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 3506.3468265909364, "interactivity_tok_s": 296.39863686751005, "output_per_gpu": 145.87832444895602, "total_per_gpu": null, "tpot_error_pct": 8.7, "tpot_ms": 3.3738346794320755, "tpot_relative": 0.912982, "ttft_error_pct": 36.88, "ttft_ms": 54.91394953192327, "ttft_relative": 1.368764, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3099.159866474992, "interactivity_tok_s": 297.58847516349357, "output_per_gpu": 143.41919292869338, "total_per_gpu": 288.24097078918555, "tpot_error_pct": 9.07, "tpot_ms": 3.360345186252946, "tpot_relative": 0.909331, "ttft_error_pct": 53.91, "ttft_ms": 18.48905675000001, "ttft_relative": 0.460851, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 3427.7044881833717, "interactivity_tok_s": 270.60657317603335, "output_per_gpu": 129.67588745445929, "total_per_gpu": 260.6199555620595, "tpot_ms": 3.6954017349367416, "tpot_relative": 1.0, "ttft_ms": 40.119356859941036, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3801.968524403184, "interactivity_tok_s": 273.4175190927581, "output_per_gpu": 269.0711386572002, "total_per_gpu": null, "tpot_error_pct": 9.25, "tpot_ms": 3.657410115189969, "tpot_relative": 0.989719, "ttft_error_pct": 30.96, "ttft_ms": 60.43797656384528, "ttft_relative": 1.506454, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3413.2992879124895, "interactivity_tok_s": 273.6175268042626, "output_per_gpu": 266.22287046669123, "total_per_gpu": 530.479317933541, "tpot_error_pct": 9.32, "tpot_ms": 3.6547366379616784, "tpot_relative": 0.988996, "ttft_error_pct": 55.29, "ttft_ms": 20.63137072499992, "ttft_relative": 0.51425, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 3787.101399595849, "interactivity_tok_s": 248.1134664940805, "output_per_gpu": 238.49671641894605, "total_per_gpu": 475.23180571798787, "tpot_ms": 4.0304140445510965, "tpot_relative": 1.090657, "ttft_ms": 46.149631368461996, "ttft_relative": 1.150308, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4641.150534790621, "interactivity_tok_s": 223.60080021713296, "output_per_gpu": 440.83896539509726, "total_per_gpu": null, "tpot_error_pct": 5.0, "tpot_ms": 4.472255908873876, "tpot_relative": 1.210222, "ttft_error_pct": 46.78, "ttft_ms": 66.03274001264653, "ttft_relative": 1.645907, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4127.252264025024, "interactivity_tok_s": 223.04083052108018, "output_per_gpu": 434.4616153569389, "total_per_gpu": 873.5482221495439, "tpot_error_pct": 4.76, "tpot_ms": 4.483484022471336, "tpot_relative": 1.21326, "ttft_error_pct": 46.2, "ttft_ms": 24.20276891874983, "ttft_relative": 0.603269, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 4353.367609565612, "interactivity_tok_s": 212.41898367371584, "output_per_gpu": 408.1786678065085, "total_per_gpu": 820.7025361464997, "tpot_ms": 4.707677170398482, "tpot_relative": 1.273928, "ttft_ms": 44.98829132062383, "ttft_relative": 1.121361, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5271.573609716083, "interactivity_tok_s": 196.73631720716952, "output_per_gpu": 776.238805137426, "total_per_gpu": null, "tpot_error_pct": 9.63, "tpot_ms": 5.082945610631557, "tpot_relative": 1.375478, "ttft_error_pct": 2.34, "ttft_ms": 71.72025004000017, "ttft_relative": 1.787672, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4784.438438437507, "interactivity_tok_s": 194.135997909641, "output_per_gpu": 752.6376500289366, "total_per_gpu": 1502.894510349546, "tpot_error_pct": 8.42, "tpot_ms": 5.151028200681472, "tpot_relative": 1.393902, "ttft_error_pct": 58.8, "ttft_ms": 30.258348790625313, "ttft_relative": 0.754208, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5264.602330583148, "interactivity_tok_s": 177.79831159922884, "output_per_gpu": 680.0931558115767, "total_per_gpu": 1358.0349991210514, "tpot_ms": 5.624350372089458, "tpot_relative": 1.521986, "ttft_ms": 73.43951394141186, "ttft_relative": 1.830526, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6558.748142300626, "interactivity_tok_s": 157.843504645822, "output_per_gpu": 1247.799095564795, "total_per_gpu": null, "tpot_error_pct": 10.57, "tpot_ms": 6.335388980647987, "tpot_relative": 1.714398, "ttft_error_pct": 21.2, "ttft_ms": 77.64521509773454, "ttft_relative": 1.935355, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6057.917389437484, "interactivity_tok_s": 153.16396074303972, "output_per_gpu": 1182.9337263657333, "total_per_gpu": 2366.4008106504025, "tpot_error_pct": 7.83, "tpot_ms": 6.528951034882684, "tpot_relative": 1.766777, "ttft_error_pct": 57.15, "ttft_ms": 42.22061264843727, "ttft_relative": 1.052375, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 6624.6869351147325, "interactivity_tok_s": 141.16384802244096, "output_per_gpu": 1080.8982827282748, "total_per_gpu": 2162.2839179140947, "tpot_ms": 7.0839667096708006, "tpot_relative": 1.916968, "ttft_ms": 98.53513020498212, "ttft_relative": 2.45605, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.65, "tpot_shape_error_pct": 4.74, "ttft_mape_pct": 40.43, "ttft_shape_error_pct": 45.51}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.23, "tpot_shape_error_pct": 5.45, "ttft_mape_pct": 60.37, "ttft_shape_error_pct": 33.05}, "framework": "vllm", "id": "e07ab2250e8c5e4d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 3847.0211730845463, "interactivity_tok_s": 270.4002400271035, "output_per_gpu": 265.9200337022729, "total_per_gpu": null, "tpot_error_pct": 8.03, "tpot_ms": 3.6982215692551352, "tpot_relative": 0.919672, "ttft_error_pct": 33.05, "ttft_ms": 63.740507736542675, "ttft_relative": 1.330453, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3393.268139474995, "interactivity_tok_s": 272.2497863814186, "output_per_gpu": 262.2387119025354, "total_per_gpu": 527.0420182525635, "tpot_error_pct": 8.66, "tpot_ms": 3.6730974642492917, "tpot_relative": 0.913424, "ttft_error_pct": 46.12, "ttft_ms": 25.812587824999987, "ttft_relative": 0.538785, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 3734.525904012844, "interactivity_tok_s": 248.6794603762674, "output_per_gpu": 238.33577760483513, "total_per_gpu": 479.0023880888044, "tpot_ms": 4.021240831417835, "tpot_relative": 1.0, "ttft_ms": 47.90886249393225, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4658.2362808252, "interactivity_tok_s": 222.9698019313586, "output_per_gpu": 439.2220309695312, "total_per_gpu": null, "tpot_error_pct": 3.56, "tpot_ms": 4.484912267661477, "tpot_relative": 1.115306, "ttft_error_pct": 29.56, "ttft_ms": 70.17103100750887, "ttft_relative": 1.464677, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4173.105949112512, "interactivity_tok_s": 224.02942348519264, "output_per_gpu": 436.6161801393961, "total_per_gpu": 870.0073477273763, "tpot_error_pct": 4.01, "tpot_ms": 4.463699385746514, "tpot_relative": 1.11003, "ttft_error_pct": 46.45, "ttft_ms": 29.00450733750035, "ttft_relative": 0.60541, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 4370.804396551102, "interactivity_tok_s": 215.03540242060387, "output_per_gpu": 413.8175015393172, "total_per_gpu": 824.5783902063545, "tpot_ms": 4.650397045059702, "tpot_relative": 1.156458, "ttft_ms": 54.16198514867574, "ttft_relative": 1.130521, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5458.458030548803, "interactivity_tok_s": 190.08532911442072, "output_per_gpu": 749.662263060138, "total_per_gpu": null, "tpot_error_pct": 1.45, "tpot_ms": 5.260795268413672, "tpot_relative": 1.308252, "ttft_error_pct": 98.36, "ttft_ms": 76.66447096161764, "ttft_relative": 1.600215, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4855.700630337482, "interactivity_tok_s": 189.79224234934145, "output_per_gpu": 738.7259842874439, "total_per_gpu": 1485.315957543917, "tpot_error_pct": 1.3, "tpot_ms": 5.26891925413552, "tpot_relative": 1.310272, "ttft_error_pct": 99.28, "ttft_ms": 33.72439810000007, "ttft_relative": 0.703928, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 9563.770390309219, "interactivity_tok_s": 187.33147727240248, "output_per_gpu": 359.7204227929618, "total_per_gpu": 725.3886986101268, "tpot_ms": 5.338131180943392, "tpot_relative": 1.327484, "ttft_ms": 4674.995496969495, "ttft_relative": 97.581016, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6507.063907449015, "interactivity_tok_s": 159.2521680058127, "output_per_gpu": 1257.7101003466862, "total_per_gpu": null, "tpot_error_pct": 13.3, "tpot_ms": 6.279349364735179, "tpot_relative": 1.561545, "ttft_error_pct": 6.8, "ttft_ms": 83.28950732492676, "ttft_relative": 1.738499, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5887.553966818742, "interactivity_tok_s": 157.9000397803986, "output_per_gpu": 1225.393047953668, "total_per_gpu": 2446.9098572457283, "tpot_error_pct": 12.55, "tpot_ms": 6.333120633729809, "tpot_relative": 1.574917, "ttft_error_pct": 52.97, "ttft_ms": 42.02423212499995, "ttft_relative": 0.87717, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 6774.222213079338, "interactivity_tok_s": 138.07921549411716, "output_per_gpu": 1060.5259064715415, "total_per_gpu": 2117.6970921641732, "tpot_ms": 7.242219594176394, "tpot_relative": 1.800991, "ttft_ms": 89.36411198810674, "ttft_relative": 1.865294, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9141.48015822493, "interactivity_tok_s": 113.0229419202242, "output_per_gpu": 1790.5196660382294, "total_per_gpu": null, "tpot_error_pct": 6.93, "tpot_ms": 8.847761197950742, "tpot_relative": 2.200256, "ttft_error_pct": 34.41, "ttft_ms": 90.22045272131992, "ttft_relative": 1.883168, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8412.099996315677, "interactivity_tok_s": 110.30849725940901, "output_per_gpu": 1712.031820458829, "total_per_gpu": 3424.835557136259, "tpot_error_pct": 4.64, "tpot_ms": 9.065484752714305, "tpot_relative": 2.2544, "ttft_error_pct": 57.01, "ttft_ms": 59.13889273906303, "ttft_relative": 1.234404, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 8896.48032477562, "interactivity_tok_s": 105.18904252697376, "output_per_gpu": 1616.8024442953135, "total_per_gpu": 3234.3338680489005, "tpot_ms": 9.50669362489509, "tpot_relative": 2.364119, "ttft_ms": 137.55898230883759, "ttft_relative": 2.871264, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.48, "tpot_shape_error_pct": 2.51, "ttft_mape_pct": 20.26, "ttft_shape_error_pct": 25.76}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.65, "tpot_shape_error_pct": 3.42, "ttft_mape_pct": 46.2, "ttft_shape_error_pct": 11.19}, "framework": "vllm", "id": "f6955cdfbb5f0dab", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 5093.133320354885, "interactivity_tok_s": 204.00542585771157, "output_per_gpu": 401.7173459455871, "total_per_gpu": null, "tpot_error_pct": 0.76, "tpot_ms": 4.901830408655276, "tpot_relative": 1.007611, "ttft_error_pct": 18.32, "ttft_ms": 78.56081230053682, "ttft_relative": 1.183153, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4482.11523805001, "interactivity_tok_s": 206.33957407288844, "output_per_gpu": 398.68876148706596, "total_per_gpu": 801.2765467931915, "tpot_error_pct": 0.38, "tpot_ms": 4.846380072718164, "tpot_relative": 0.996212, "ttft_error_pct": 41.36, "ttft_ms": 38.93710490000011, "ttft_relative": 0.586406, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 4526.202545547858, "interactivity_tok_s": 205.55801907703682, "output_per_gpu": 394.0589411414046, "total_per_gpu": 791.9716282271216, "tpot_ms": 4.8648065616220535, "tpot_relative": 1.0, "ttft_ms": 66.39956175349653, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6435.023227824629, "interactivity_tok_s": 161.14016882824865, "output_per_gpu": 635.895140565531, "total_per_gpu": null, "tpot_error_pct": 3.48, "tpot_ms": 6.205777288627832, "tpot_relative": 1.275647, "ttft_error_pct": 17.15, "ttft_ms": 86.51306155835732, "ttft_relative": 1.302916, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5770.219716975005, "interactivity_tok_s": 162.14678793320127, "output_per_gpu": 632.9959543327842, "total_per_gpu": 1261.3163606887003, "tpot_error_pct": 2.84, "tpot_ms": 6.167251369863488, "tpot_relative": 1.267728, "ttft_error_pct": 40.34, "ttft_ms": 44.05615837500019, "ttft_relative": 0.663501, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5639.59795688279, "interactivity_tok_s": 166.75376716995495, "output_per_gpu": 646.299386936397, "total_per_gpu": 1287.8249617017086, "tpot_ms": 5.996866019709185, "tpot_relative": 1.232704, "ttft_ms": 73.84968311525881, "ttft_relative": 1.112201, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8286.374525452398, "interactivity_tok_s": 124.88112995052794, "output_per_gpu": 987.6454382869197, "total_per_gpu": null, "tpot_error_pct": 0.97, "tpot_ms": 8.007614924657979, "tpot_relative": 1.646029, "ttft_error_pct": 1.61, "ttft_ms": 94.58445752728551, "ttft_relative": 1.424474, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7375.311726506258, "interactivity_tok_s": 124.97071264151505, "output_per_gpu": 974.9080898999988, "total_per_gpu": 1960.1944074890991, "tpot_error_pct": 0.89, "tpot_ms": 8.001874830213634, "tpot_relative": 1.64485, "ttft_error_pct": 46.1, "ttft_ms": 51.81607375625039, "ttft_relative": 0.780368, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 7355.675572832115, "interactivity_tok_s": 126.08720366635569, "output_per_gpu": 974.043884161711, "total_per_gpu": 1958.4567962489612, "tpot_ms": 7.931018937069453, "tpot_relative": 1.630285, "ttft_ms": 96.12944034743123, "ttft_relative": 1.447742, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10315.026264939956, "interactivity_tok_s": 100.17496549759572, "output_per_gpu": 1586.8112770234695, "total_per_gpu": null, "tpot_error_pct": 5.98, "tpot_ms": 9.982534009697272, "tpot_relative": 2.05199, "ttft_error_pct": 21.82, "ttft_ms": 102.89397301964682, "ttft_relative": 1.549618, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9349.107711115676, "interactivity_tok_s": 99.41574065886947, "output_per_gpu": 1548.0624867411589, "total_per_gpu": 3091.228047004963, "tpot_error_pct": 5.27, "tpot_ms": 10.058769299233543, "tpot_relative": 2.067661, "ttft_error_pct": 50.58, "ttft_ms": 65.04472768124997, "ttft_relative": 0.979596, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 9931.77568797837, "interactivity_tok_s": 94.17995425042028, "output_per_gpu": 1453.7599893653344, "total_per_gpu": 2902.9213557134362, "tpot_ms": 10.617970755656186, "tpot_relative": 2.182609, "ttft_ms": 131.61446746380534, "ttft_relative": 1.982159, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15126.760499371305, "interactivity_tok_s": 68.13330597081006, "output_per_gpu": 2164.111741001027, "total_per_gpu": null, "tpot_error_pct": 1.19, "tpot_ms": 14.677109612564875, "tpot_relative": 3.016998, "ttft_error_pct": 42.39, "ttft_ms": 112.077365717438, "ttft_relative": 1.687923, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13970.892641218801, "interactivity_tok_s": 66.38549512085437, "output_per_gpu": 2069.347301633122, "total_per_gpu": 4139.627624910445, "tpot_error_pct": 3.85, "tpot_ms": 15.063531546755906, "tpot_relative": 3.09643, "ttft_error_pct": 52.63, "ttft_ms": 92.14782414999988, "ttft_relative": 1.387778, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 13557.89659051079, "interactivity_tok_s": 68.94258464173099, "output_per_gpu": 2129.4002790526642, "total_per_gpu": 4259.7606562715255, "tpot_ms": 14.504823182893833, "tpot_relative": 2.981583, "ttft_ms": 194.53692254028283, "ttft_relative": 2.929792, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp4"], "rows": 96}, {"aic": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.76, "tpot_shape_error_pct": 6.28, "ttft_mape_pct": 22.13, "ttft_shape_error_pct": 15.02}, "aisimulate": {"coverage_pct": 100.0, "points": 13, "status_counts": {"failed": 0, "success": 13, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.68, "tpot_shape_error_pct": 6.4, "ttft_mape_pct": 44.6, "ttft_shape_error_pct": 6.92}, "gpu_skus": ["b200", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.02, "tpot_shape_error_pct": 7.89, "ttft_mape_pct": 24.3, "ttft_shape_error_pct": 9.79}, "aisimulate": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.17, "tpot_shape_error_pct": 8.06, "ttft_mape_pct": 47.0, "ttft_shape_error_pct": 6.76}, "gpu": "b200", "precisions": ["fp4"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.58, "tpot_shape_error_pct": 3.62, "ttft_mape_pct": 9.68, "ttft_shape_error_pct": 5.94}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.96, "tpot_shape_error_pct": 3.88, "ttft_mape_pct": 38.09, "ttft_shape_error_pct": 5.54}, "framework": "trt", "id": "9b34c34204a5070c", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 27440.09564310855, "interactivity_tok_s": 298.9453851156323, "output_per_gpu": 1193.4949981493987, "total_per_gpu": null, "tpot_error_pct": 3.94, "tpot_ms": 3.345092614870771, "tpot_relative": 0.960578, "ttft_error_pct": 14.79, "ttft_ms": 40.442034702068376, "ttft_relative": 1.147916, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24245.8383117001, "interactivity_tok_s": 299.7832645648632, "output_per_gpu": 1181.467345233806, "total_per_gpu": 1332.2327336335236, "tpot_error_pct": 4.21, "tpot_ms": 3.335743245879668, "tpot_relative": 0.957893, "ttft_error_pct": 35.48, "ttft_ms": 22.73066989999898, "ttft_relative": 0.645193, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 25317.097040463705, "interactivity_tok_s": 287.1602780769421, "output_per_gpu": 1130.2544721654335, "total_per_gpu": 1274.4846577680812, "tpot_ms": 3.482375789217124, "tpot_relative": 1.0, "ttft_ms": 35.23081993917003, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36266.43910962046, "interactivity_tok_s": 226.1341388638213, "output_per_gpu": 1806.5499906315897, "total_per_gpu": null, "tpot_error_pct": 4.61, "tpot_ms": 4.422154058756264, "tpot_relative": 1.269867, "ttft_error_pct": 11.96, "ttft_ms": 44.57521434790733, "ttft_relative": 1.265234, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32318.01510153747, "interactivity_tok_s": 227.8114871709218, "output_per_gpu": 1785.175244797042, "total_per_gpu": 2008.9195945445813, "tpot_error_pct": 5.32, "tpot_ms": 4.38959427559385, "tpot_relative": 1.260517, "ttft_error_pct": 35.34, "ttft_ms": 25.743008262500734, "ttft_relative": 0.730696, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 34141.32299651683, "interactivity_tok_s": 215.70247690271424, "output_per_gpu": 1687.1839783466971, "total_per_gpu": 1898.646625075571, "tpot_ms": 4.636015378029332, "tpot_relative": 1.331279, "ttft_ms": 39.81453219312243, "ttft_relative": 1.130105, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45899.54270631132, "interactivity_tok_s": 178.66094056297683, "output_per_gpu": 2855.5405596748874, "total_per_gpu": null, "tpot_error_pct": 0.52, "tpot_ms": 5.597194310345111, "tpot_relative": 1.607292, "ttft_error_pct": 5.79, "ttft_ms": 52.92411027450986, "ttft_relative": 1.502211, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41048.5141111559, "interactivity_tok_s": 179.83904045449214, "output_per_gpu": 2818.2965567209567, "total_per_gpu": 3171.8811911735365, "tpot_error_pct": 0.13, "tpot_ms": 5.560527889121204, "tpot_relative": 1.596763, "ttft_error_pct": 38.14, "ttft_ms": 30.94762989374931, "ttft_relative": 0.878425, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 41116.14400517574, "interactivity_tok_s": 179.59692967573488, "output_per_gpu": 2809.462051208053, "total_per_gpu": 3161.938304998953, "tpot_ms": 5.56802391781149, "tpot_relative": 1.598915, "ttft_ms": 50.02598481805762, "ttft_relative": 1.419949, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 57328.18901640455, "interactivity_tok_s": 143.05507323203756, "output_per_gpu": 4573.980522244769, "total_per_gpu": null, "tpot_error_pct": 9.24, "tpot_ms": 6.990314830554694, "tpot_relative": 2.007341, "ttft_error_pct": 6.17, "ttft_ms": 70.52023933104614, "ttft_relative": 2.001663, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 51329.001116196, "interactivity_tok_s": 144.56679067698684, "output_per_gpu": 4524.731524642304, "total_per_gpu": 5086.523216986133, "tpot_error_pct": 10.19, "tpot_ms": 6.917217953840813, "tpot_relative": 1.98635, "ttft_error_pct": 43.41, "ttft_ms": 37.583837493749535, "ttft_relative": 1.066789, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 57197.007735991065, "interactivity_tok_s": 129.84043662921042, "output_per_gpu": 4056.5733327105636, "total_per_gpu": 4560.238397762235, "tpot_ms": 7.701760914866088, "tpot_relative": 2.21164, "ttft_ms": 66.41958834807156, "ttft_relative": 1.885269, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.24, "tpot_shape_error_pct": 1.37, "ttft_mape_pct": 32.24, "ttft_shape_error_pct": 3.66}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.33, "tpot_shape_error_pct": 1.54, "ttft_mape_pct": 61.96, "ttft_shape_error_pct": 7.84}, "framework": "trt", "id": "89391eb978a24020", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 25482.759228567993, "interactivity_tok_s": 321.87178000115483, "output_per_gpu": 1286.2743896192317, "total_per_gpu": null, "tpot_error_pct": 12.83, "tpot_ms": 3.1068271968310244, "tpot_relative": 0.871665, "ttft_error_pct": 30.97, "ttft_ms": 34.73765932507508, "ttft_relative": 0.690262, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23011.87604284347, "interactivity_tok_s": 322.4890405891673, "output_per_gpu": 1256.6178310596742, "total_per_gpu": 1412.6397859747242, "tpot_error_pct": 13.0, "tpot_ms": 3.1008805699972397, "tpot_relative": 0.869996, "ttft_error_pct": 63.4, "ttft_ms": 18.420353368750842, "ttft_relative": 0.366025, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19349737828", "measured": {"e2e_ms": 26483.45075109537, "interactivity_tok_s": 280.564243196496, "output_per_gpu": 1091.8110795828272, "total_per_gpu": 1227.3705908550608, "tpot_ms": 3.564246065738462, "tpot_relative": 1.0, "ttft_ms": 50.32534157653572, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31277.131380654566, "interactivity_tok_s": 262.22569788633666, "output_per_gpu": 2096.269861796189, "total_per_gpu": null, "tpot_error_pct": 11.64, "tpot_ms": 3.8135087753049137, "tpot_relative": 1.069934, "ttft_error_pct": 33.5, "ttft_ms": 40.681002132016225, "ttft_relative": 0.80836, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28182.987327924744, "interactivity_tok_s": 262.27789529762225, "output_per_gpu": 2034.2816010639556, "total_per_gpu": 2288.2800869476528, "tpot_error_pct": 11.66, "tpot_ms": 3.8127498272976488, "tpot_relative": 1.069721, "ttft_error_pct": 60.53, "ttft_ms": 24.147335984374813, "ttft_relative": 0.479825, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19349737828", "measured": {"e2e_ms": 31940.070099128934, "interactivity_tok_s": 231.69760839014165, "output_per_gpu": 1797.4917726065448, "total_per_gpu": 2021.9249033941792, "tpot_ms": 4.315970315568213, "tpot_relative": 1.210907, "ttft_ms": 61.174485991796246, "ttft_relative": 1.21558, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.79, "tpot_shape_error_pct": 17.54, "ttft_mape_pct": 38.51, "ttft_shape_error_pct": 18.64}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.66, "tpot_shape_error_pct": 17.59, "ttft_mape_pct": 48.91, "ttft_shape_error_pct": 8.03}, "framework": "vllm", "id": "a0be98947fb1bbc5", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 27857.297734167063, "interactivity_tok_s": 294.6786866749677, "output_per_gpu": 588.0685253942426, "total_per_gpu": null, "tpot_error_pct": 10.9, "tpot_ms": 3.3935267300244414, "tpot_relative": 0.891012, "ttft_error_pct": 58.16, "ttft_ms": 60.92028853686655, "ttft_relative": 1.581643, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 24955.83543539991, "interactivity_tok_s": 295.81517391580417, "output_per_gpu": 576.6143369721141, "total_per_gpu": 648.9566066570386, "tpot_error_pct": 11.24, "tpot_ms": 3.380489197909175, "tpot_relative": 0.887589, "ttft_error_pct": 51.51, "ttft_ms": 18.67630928749883, "ttft_relative": 0.484884, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 28136.05487116729, "interactivity_tok_s": 262.5623686008992, "output_per_gpu": 510.4415186006862, "total_per_gpu": 574.4817195275305, "tpot_ms": 3.808618901972288, "tpot_relative": 1.0, "ttft_ms": 38.51708140282426, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35436.67421402027, "interactivity_tok_s": 231.57637932220902, "output_per_gpu": 924.5788643178358, "total_per_gpu": null, "tpot_error_pct": 2.68, "tpot_ms": 4.318229704285287, "tpot_relative": 1.133805, "ttft_error_pct": 44.22, "ttft_ms": 66.0547062194878, "ttft_relative": 1.714946, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 31882.889642363392, "interactivity_tok_s": 232.73247313789886, "output_per_gpu": 909.8700330992002, "total_per_gpu": 1022.8397027743848, "tpot_error_pct": 2.17, "tpot_ms": 4.2967789862632495, "tpot_relative": 1.128172, "ttft_error_pct": 48.35, "ttft_ms": 23.656774706250637, "ttft_relative": 0.614189, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 31234.948615117173, "interactivity_tok_s": 237.776232217959, "output_per_gpu": 923.7591096355014, "total_per_gpu": 1038.4532502035836, "tpot_ms": 4.2056348133372055, "tpot_relative": 1.104241, "ttft_ms": 45.80042221205076, "ttft_relative": 1.189094, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 47940.13119733471, "interactivity_tok_s": 171.11379689451763, "output_per_gpu": 1366.8715200271106, "total_per_gpu": null, "tpot_error_pct": 6.79, "tpot_ms": 5.844064114926079, "tpot_relative": 1.534431, "ttft_error_pct": 13.13, "ttft_ms": 71.40203197520437, "ttft_relative": 1.853776, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43106.137238058305, "interactivity_tok_s": 171.45637590403425, "output_per_gpu": 1337.166138139075, "total_per_gpu": 1504.1234435016365, "tpot_error_pct": 6.57, "tpot_ms": 5.8323873622507305, "tpot_relative": 1.531365, "ttft_error_pct": 46.88, "ttft_ms": 33.52426090937455, "ttft_relative": 0.870374, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 40480.433687969344, "interactivity_tok_s": 182.72831529553048, "output_per_gpu": 1415.6489768635029, "total_per_gpu": 1592.4055755950026, "tpot_ms": 5.472605591435998, "tpot_relative": 1.4369, "ttft_ms": 63.1143780425191, "ttft_relative": 1.638607, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.92, "tpot_shape_error_pct": 3.06, "ttft_mape_pct": 17.26, "ttft_shape_error_pct": 25.48}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.33, "tpot_shape_error_pct": 3.07, "ttft_mape_pct": 39.17, "ttft_shape_error_pct": 7.23}, "gpu": "h200", "precisions": ["fp4"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.92, "tpot_shape_error_pct": 3.06, "ttft_mape_pct": 17.26, "ttft_shape_error_pct": 25.48}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.33, "tpot_shape_error_pct": 3.07, "ttft_mape_pct": 39.17, "ttft_shape_error_pct": 7.23}, "framework": "vllm", "id": "e6e572a875e3d018", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 73829.46828301996, "interactivity_tok_s": 111.12120355918341, "output_per_gpu": 887.5588775582552, "total_per_gpu": null, "tpot_error_pct": 3.74, "tpot_ms": 8.999182585953523, "tpot_relative": 1.037432, "ttft_error_pct": 6.0, "ttft_ms": 117.16372147464841, "ttft_relative": 1.059972, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 65498.74334200039, "interactivity_tok_s": 112.43888112704302, "output_per_gpu": 882.3530506183906, "total_per_gpu": 992.9424788180933, "tpot_error_pct": 2.53, "tpot_ms": 8.893720659405307, "tpot_relative": 1.025274, "ttft_error_pct": 35.69, "ttft_ms": 71.09010945000101, "ttft_relative": 0.643147, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 63951.54242207063, "interactivity_tok_s": 115.28065197053566, "output_per_gpu": 901.6807894031466, "total_per_gpu": 1014.6926533603971, "tpot_ms": 8.674482516420778, "tpot_relative": 1.0, "ttft_ms": 110.53470150800422, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 99181.38230811954, "interactivity_tok_s": 82.69318143883243, "output_per_gpu": 1321.3770261121983, "total_per_gpu": null, "tpot_error_pct": 3.05, "tpot_ms": 12.092895479413778, "tpot_relative": 1.394077, "ttft_error_pct": 3.73, "ttft_ms": 128.47543624128414, "ttft_relative": 1.162309, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 88329.46997231217, "interactivity_tok_s": 83.59001955819865, "output_per_gpu": 1312.4517051638506, "total_per_gpu": 1477.1124309133636, "tpot_error_pct": 1.94, "tpot_ms": 11.963150688148371, "tpot_relative": 1.37912, "ttft_error_pct": 37.52, "ttft_ms": 83.3820323062525, "ttft_relative": 0.754352, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 86697.92779879353, "interactivity_tok_s": 85.21266086700159, "output_per_gpu": 1335.2474927818448, "total_per_gpu": 1502.768187335118, "tpot_ms": 11.735345309317148, "tpot_relative": 1.352858, "ttft_ms": 133.45203361022868, "ttft_relative": 1.207332, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 128320.24692140786, "interactivity_tok_s": 63.902607275627176, "output_per_gpu": 2042.639460945983, "total_per_gpu": null, "tpot_error_pct": 3.12, "tpot_ms": 15.64881375945682, "tpot_relative": 1.804005, "ttft_error_pct": 19.01, "ttft_ms": 140.8134176970431, "ttft_relative": 1.27393, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 114712.08568330322, "interactivity_tok_s": 64.69905591025744, "output_per_gpu": 2027.4989427128025, "total_per_gpu": 2279.233671292526, "tpot_error_pct": 4.31, "tpot_ms": 15.456176074455811, "tpot_relative": 1.781798, "ttft_error_pct": 40.09, "ttft_ms": 104.16795488750094, "ttft_relative": 0.9424, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 119972.04578428282, "interactivity_tok_s": 61.908595669115954, "output_per_gpu": 1936.3278833252778, "total_per_gpu": 2176.742792493142, "tpot_ms": 16.152845807466203, "tpot_relative": 1.862111, "ttft_ms": 173.8758646264614, "ttft_relative": 1.573043, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 195575.42801383487, "interactivity_tok_s": 41.91480182112049, "output_per_gpu": 2680.4185235525433, "total_per_gpu": null, "tpot_error_pct": 1.78, "tpot_ms": 23.857920270449878, "tpot_relative": 2.750357, "ttft_error_pct": 40.28, "ttft_ms": 155.20307857990173, "ttft_relative": 1.404112, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 174166.44808882658, "interactivity_tok_s": 42.43853054960342, "output_per_gpu": 2660.592546200374, "total_per_gpu": 2992.791627160896, "tpot_error_pct": 0.52, "tpot_ms": 23.563492586793743, "tpot_relative": 2.716415, "ttft_error_pct": 43.4, "ttft_ms": 147.08983632968653, "ttft_relative": 1.330712, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019467297", "measured": {"e2e_ms": 173365.2093521967, "interactivity_tok_s": 42.661182239860004, "output_per_gpu": 2672.622216361086, "total_per_gpu": 3006.323310614607, "tpot_ms": 23.440513072927946, "tpot_relative": 2.702238, "ttft_ms": 259.88767307026137, "ttft_relative": 2.351186, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}]}], "identity": "1024:8192", "label": "1k8k", "precisions": ["fp4"], "rows": 13}, {"aic": {"coverage_pct": 100.0, "points": 98, "status_counts": {"failed": 0, "success": 98, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.91, "tpot_shape_error_pct": 12.16, "ttft_mape_pct": 41.87, "ttft_shape_error_pct": 46.4}, "aisimulate": {"coverage_pct": 100.0, "points": 98, "status_counts": {"failed": 0, "success": 98, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.8, "tpot_shape_error_pct": 11.85, "ttft_mape_pct": 46.81, "ttft_shape_error_pct": 38.74}, "gpu_skus": ["b200", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 46, "status_counts": {"failed": 0, "success": 46, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.32, "tpot_shape_error_pct": 7.6, "ttft_mape_pct": 26.95, "ttft_shape_error_pct": 32.28}, "aisimulate": {"coverage_pct": 100.0, "points": 46, "status_counts": {"failed": 0, "success": 46, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.89, "tpot_shape_error_pct": 8.32, "ttft_mape_pct": 29.43, "ttft_shape_error_pct": 18.64}, "gpu": "b200", "precisions": ["fp4"], "rows": 46, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.56, "tpot_shape_error_pct": 2.08, "ttft_mape_pct": 26.06, "ttft_shape_error_pct": 31.88}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.71, "tpot_shape_error_pct": 2.61, "ttft_mape_pct": 30.34, "ttft_shape_error_pct": 52.27}, "framework": "trt", "id": "9f49b8bc80895711", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 3215.6571190055047, "interactivity_tok_s": 329.41923301233174, "output_per_gpu": 627.4821877871049, "total_per_gpu": null, "tpot_error_pct": 3.23, "tpot_ms": 3.03564546263929, "tpot_relative": 1.032297, "ttft_error_pct": 9.67, "ttft_ms": 110.19181072551099, "ttft_relative": 0.903325, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2925.298666249996, "interactivity_tok_s": 320.6171912521793, "output_per_gpu": 614.4509017607539, "total_per_gpu": 5523.547156692141, "tpot_error_pct": 6.06, "tpot_ms": 3.1189843442095926, "tpot_relative": 1.060637, "ttft_error_pct": 51.89, "ttft_ms": 58.682755374999985, "ttft_relative": 0.481067, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2823.984550742898, "interactivity_tok_s": 340.0585126824474, "output_per_gpu": 636.9799092138988, "total_per_gpu": 5726.069497703152, "tpot_ms": 2.94067039260922, "tpot_relative": 1.0, "ttft_ms": 121.98465588735417, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3939.883839993659, "interactivity_tok_s": 267.87789869423545, "output_per_gpu": 1030.0184585190405, "total_per_gpu": null, "tpot_error_pct": 6.03, "tpot_ms": 3.733044065503263, "tpot_relative": 1.269453, "ttft_error_pct": 14.74, "ttft_ms": 120.97976098382125, "ttft_relative": 0.991762, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3591.5310865249885, "interactivity_tok_s": 262.77139471981525, "output_per_gpu": 1014.774300449706, "total_per_gpu": 9026.886572408179, "tpot_error_pct": 8.09, "tpot_ms": 3.8055892692059126, "tpot_relative": 1.294123, "ttft_error_pct": 35.2, "ttft_ms": 68.32254682500006, "ttft_relative": 0.560091, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 3364.2009810137097, "interactivity_tok_s": 284.0398303852348, "output_per_gpu": 1082.9818781820077, "total_per_gpu": 9633.624511371894, "tpot_ms": 3.520632999406209, "tpot_relative": 1.197221, "ttft_ms": 105.4353361716494, "ttft_relative": 0.864333, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4872.484037913997, "interactivity_tok_s": 216.29191789268185, "output_per_gpu": 1676.3302025748858, "total_per_gpu": null, "tpot_error_pct": 0.1, "tpot_ms": 4.623381260580309, "tpot_relative": 1.57222, "ttft_error_pct": 13.14, "ttft_ms": 142.76500834034184, "ttft_relative": 1.170352, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4370.8143385812555, "interactivity_tok_s": 213.5933428387084, "output_per_gpu": 1626.7999651171388, "total_per_gpu": 14681.462501903787, "tpot_error_pct": 1.16, "tpot_ms": 4.681793855134961, "tpot_relative": 1.592084, "ttft_error_pct": 30.79, "ttft_ms": 87.34037158125005, "ttft_relative": 0.715995, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 4361.50592136255, "interactivity_tok_s": 216.07519593138045, "output_per_gpu": 1631.5026353374737, "total_per_gpu": 14723.902923577694, "tpot_ms": 4.628018480740254, "tpot_relative": 1.573797, "ttft_ms": 126.18746852676848, "ttft_relative": 1.034454, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6746.343638338088, "interactivity_tok_s": 155.96002000141104, "output_per_gpu": 2439.659705188194, "total_per_gpu": null, "tpot_error_pct": 4.22, "tpot_ms": 6.411899664997174, "tpot_relative": 2.180421, "ttft_error_pct": 9.39, "ttft_ms": 186.97028104597817, "ttft_relative": 1.532736, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6096.547544278104, "interactivity_tok_s": 154.89957074531605, "output_per_gpu": 2372.3704743291396, "total_per_gpu": 21214.49216078313, "tpot_error_pct": 4.93, "tpot_ms": 6.455795811365983, "tpot_relative": 2.195348, "ttft_error_pct": 28.82, "ttft_ms": 121.65530314375003, "ttft_relative": 0.9973, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 5865.61343003732, "interactivity_tok_s": 162.5363980976485, "output_per_gpu": 2463.634408570957, "total_per_gpu": 22030.60331984768, "tpot_ms": 6.152468073023378, "tpot_relative": 2.092199, "ttft_ms": 170.92277605297568, "ttft_relative": 1.401183, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10157.506966570927, "interactivity_tok_s": 102.94568555749636, "output_per_gpu": 3246.645615542652, "total_per_gpu": null, "tpot_error_pct": 6.93, "tpot_ms": 9.713860222354713, "tpot_relative": 3.303281, "ttft_error_pct": 17.05, "ttft_ms": 220.22795910205653, "ttft_relative": 1.805374, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9206.179028560962, "interactivity_tok_s": 102.32078562935077, "output_per_gpu": 3151.1200886636984, "total_per_gpu": 28403.49685467294, "tpot_error_pct": 7.59, "tpot_ms": 9.77318531957352, "tpot_relative": 3.323455, "ttft_error_pct": 26.23, "ttft_ms": 195.84976115624994, "ttft_relative": 1.605528, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 8641.345873475439, "interactivity_tok_s": 110.08218292898135, "output_per_gpu": 3355.279654968158, "total_per_gpu": 30243.745857001264, "tpot_ms": 9.084122183924551, "tpot_relative": 3.089133, "ttft_ms": 265.4917608353571, "ttft_relative": 2.176436, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15104.239777681696, "interactivity_tok_s": 68.75002922387854, "output_per_gpu": 4359.252892758042, "total_per_gpu": null, "tpot_error_pct": 2.35, "tpot_ms": 14.545448362553946, "tpot_relative": 4.946304, "ttft_error_pct": 48.19, "ttft_ms": 224.24610278901034, "ttft_relative": 1.838314, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13816.28763652115, "interactivity_tok_s": 68.32205005139095, "output_per_gpu": 4200.291260670453, "total_per_gpu": 37920.57327487395, "tpot_error_pct": 3.0, "tpot_ms": 14.636563148322002, "tpot_relative": 4.977288, "ttft_error_pct": 21.13, "ttft_ms": 341.3813926382806, "ttft_relative": 2.79856, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 13515.310592342332, "interactivity_tok_s": 70.36898024305967, "output_per_gpu": 4294.091812663117, "total_per_gpu": 38767.41233538559, "tpot_ms": 14.210807042334933, "tpot_relative": 4.832506, "ttft_ms": 432.8474949681379, "ttft_relative": 3.548377, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24830.736755963255, "interactivity_tok_s": 41.59118874783808, "output_per_gpu": 5296.208011061807, "total_per_gpu": null, "tpot_error_pct": 2.04, "tpot_ms": 24.043554178334954, "tpot_relative": 8.176215, "ttft_error_pct": 70.27, "ttft_ms": 234.18083152659682, "ttft_relative": 1.919756, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22851.349427319365, "interactivity_tok_s": 41.55545995901584, "output_per_gpu": 5106.894012887014, "total_per_gpu": 45948.83132398518, "tpot_error_pct": 2.13, "tpot_ms": 24.064226481580327, "tpot_relative": 8.183245, "ttft_error_pct": 18.31, "ttft_ms": 643.4418835609388, "ttft_relative": 5.274777, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 22529.02977845606, "interactivity_tok_s": 42.440169910542096, "output_per_gpu": 5178.470194673922, "total_per_gpu": 46592.83174683365, "tpot_ms": 23.56258238616526, "tpot_relative": 8.012657, "ttft_ms": 787.6301363097809, "ttft_relative": 6.456797, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 3.95, "tpot_shape_error_pct": 4.46, "ttft_mape_pct": 32.83, "ttft_shape_error_pct": 35.24}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.03, "tpot_shape_error_pct": 3.27, "ttft_mape_pct": 24.56, "ttft_shape_error_pct": 7.59}, "framework": "trt", "id": "25738a8a9de29ef6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 3674.0885065286507, "interactivity_tok_s": 289.78949472989007, "output_per_gpu": 1095.9077946187474, "total_per_gpu": null, "tpot_error_pct": 0.12, "tpot_ms": 3.450780715609067, "tpot_relative": 1.001176, "ttft_error_pct": 33.62, "ttft_ms": 143.93983446057516, "ttft_relative": 1.336156, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3341.010913174958, "interactivity_tok_s": 281.51745111681316, "output_per_gpu": 1078.2309917786533, "total_per_gpu": 9692.653573833075, "tpot_error_pct": 3.06, "tpot_ms": 3.5521776573099864, "tpot_relative": 1.030594, "ttft_error_pct": 29.17, "ttft_ms": 76.30332542499995, "ttft_relative": 0.708304, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 3274.650181450852, "interactivity_tok_s": 290.1303031186392, "output_per_gpu": 1100.0622181680137, "total_per_gpu": 9888.903279227772, "tpot_ms": 3.4467271748276604, "tpot_relative": 1.0, "ttft_ms": 107.72685264964821, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5059.032159420718, "interactivity_tok_s": 208.73909641320043, "output_per_gpu": 1604.0213076753632, "total_per_gpu": null, "tpot_error_pct": 6.89, "tpot_ms": 4.7906693915187475, "tpot_relative": 1.389918, "ttft_error_pct": 27.31, "ttft_ms": 158.1773718970387, "ttft_relative": 1.468319, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4591.358797162548, "interactivity_tok_s": 205.68742273302396, "output_per_gpu": 1590.872448254247, "total_per_gpu": 14151.5459498692, "tpot_error_pct": 8.48, "tpot_ms": 4.861745977039975, "tpot_relative": 1.41054, "ttft_error_pct": 27.75, "ttft_ms": 89.77033287500026, "ttft_relative": 0.833314, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 4273.7826260156, "interactivity_tok_s": 223.13052980158255, "output_per_gpu": 1706.6112745113744, "total_per_gpu": 15181.096320019244, "tpot_ms": 4.481681645668317, "tpot_relative": 1.300272, "ttft_ms": 124.24866264627781, "ttft_relative": 1.153368, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6949.893236522211, "interactivity_tok_s": 151.26694517481184, "output_per_gpu": 2351.2607734259013, "total_per_gpu": null, "tpot_error_pct": 5.83, "tpot_ms": 6.610829608837203, "tpot_relative": 1.918002, "ttft_error_pct": 17.7, "ttft_ms": 187.01454668175225, "ttft_relative": 1.736007, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6205.623187049967, "interactivity_tok_s": 150.24378588764807, "output_per_gpu": 2304.603199748248, "total_per_gpu": 20798.467042280197, "tpot_error_pct": 6.55, "tpot_ms": 6.655849319104602, "tpot_relative": 1.931064, "ttft_error_pct": 26.66, "ttft_ms": 116.5238437812499, "ttft_relative": 1.08166, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 5875.4600406879035, "interactivity_tok_s": 160.0908583226061, "output_per_gpu": 2427.496875379256, "total_per_gpu": 21907.55170492208, "tpot_ms": 6.246452861067533, "tpot_relative": 1.812285, "ttft_ms": 158.8904605001062, "ttft_relative": 1.474938, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9553.761715902561, "interactivity_tok_s": 109.91119338108646, "output_per_gpu": 3444.645246521732, "total_per_gpu": null, "tpot_error_pct": 3.57, "tpot_ms": 9.098254411020527, "tpot_relative": 2.639679, "ttft_error_pct": 12.67, "ttft_ms": 246.24745342856298, "ttft_relative": 2.28585, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8600.739250531284, "interactivity_tok_s": 109.68045251063319, "output_per_gpu": 3371.7550926334197, "total_per_gpu": 30151.307628703988, "tpot_error_pct": 3.79, "tpot_ms": 9.117394915042432, "tpot_relative": 2.645233, "ttft_error_pct": 25.17, "ttft_ms": 163.532242015625, "ttft_relative": 1.518027, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 8349.733143877529, "interactivity_tok_s": 113.83335240750178, "output_per_gpu": 3466.979186382667, "total_per_gpu": 31002.831795026428, "tpot_ms": 8.784771588033268, "tpot_relative": 2.548728, "ttft_ms": 218.55169775299146, "ttft_relative": 2.028758, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15415.60858564196, "interactivity_tok_s": 67.63084082199525, "output_per_gpu": 4275.389376703416, "total_per_gpu": null, "tpot_error_pct": 9.61, "tpot_ms": 14.786153592737456, "tpot_relative": 4.289911, "ttft_error_pct": 18.29, "ttft_ms": 289.3734602715416, "ttft_relative": 2.686178, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13996.025069634436, "interactivity_tok_s": 67.13777170618613, "output_per_gpu": 4151.289834529892, "total_per_gpu": 37418.80488214219, "tpot_error_pct": 10.42, "tpot_ms": 14.894745157409792, "tpot_relative": 4.321417, "ttft_error_pct": 24.66, "ttft_ms": 266.8123754437489, "ttft_relative": 2.476749, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 12792.494284513668, "interactivity_tok_s": 74.13017791679967, "output_per_gpu": 4540.658187074797, "total_per_gpu": 40928.484762831344, "tpot_ms": 13.489782813179733, "tpot_relative": 3.913795, "ttft_ms": 354.15133345741197, "ttft_relative": 3.287494, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22748.54183726874, "interactivity_tok_s": 45.56699097993173, "output_per_gpu": 5786.2680126187115, "total_per_gpu": null, "tpot_error_pct": 0.86, "tpot_ms": 21.945710666750244, "tpot_relative": 6.367116, "ttft_error_pct": 49.01, "ttft_ms": 298.0798251832407, "ttft_relative": 2.766997, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20865.435272583534, "interactivity_tok_s": 45.12463805247996, "output_per_gpu": 5565.701837276306, "total_per_gpu": 50247.61171272872, "tpot_error_pct": 1.85, "tpot_ms": 22.160842572011322, "tpot_relative": 6.429532, "ttft_error_pct": 19.82, "ttft_ms": 468.67031066953075, "ttft_relative": 4.350543, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 20610.055734404523, "interactivity_tok_s": 45.960395622733905, "output_per_gpu": 5641.703336323059, "total_per_gpu": 50933.75946647821, "tpot_ms": 21.757863187438684, "tpot_relative": 6.312615, "ttft_ms": 584.5364274708118, "ttft_relative": 5.426098, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 34907.06520347664, "interactivity_tok_s": 29.57290353138156, "output_per_gpu": 7534.006247776421, "total_per_gpu": null, "tpot_error_pct": 0.74, "tpot_ms": 33.81473851354639, "tpot_relative": 9.81068, "ttft_error_pct": 71.22, "ttft_ms": 314.5877041186842, "ttft_relative": 2.920235, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32188.986801479663, "interactivity_tok_s": 29.480746419758027, "output_per_gpu": 7243.399693578544, "total_per_gpu": 65171.85394734567, "tpot_error_pct": 1.05, "tpot_ms": 33.92044372831072, "tpot_relative": 9.841349, "ttft_error_pct": 18.7, "ttft_ms": 888.5584428390645, "ttft_relative": 8.248254, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 32067.44871478386, "interactivity_tok_s": 29.791130351151583, "output_per_gpu": 7275.321391580474, "total_per_gpu": 65459.06662757058, "tpot_ms": 33.567037846932344, "tpot_relative": 9.738815, "ttft_ms": 1092.9731557473133, "ttft_relative": 10.145782, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.92, "tpot_shape_error_pct": 7.9, "ttft_mape_pct": 16.48, "ttft_shape_error_pct": 11.09}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.44, "tpot_shape_error_pct": 8.72, "ttft_mape_pct": 45.12, "ttft_shape_error_pct": 25.05}, "framework": "trt", "id": "159c7a0ce8d69b74", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 2456.4695974476012, "interactivity_tok_s": 428.9349792565269, "output_per_gpu": 205.8401733083677, "total_per_gpu": null, "tpot_error_pct": 2.12, "tpot_ms": 2.3313556794395742, "tpot_relative": 0.978826, "ttft_error_pct": 13.84, "ttft_ms": 71.49273738091676, "ttft_relative": 0.861618, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2232.407047300005, "interactivity_tok_s": 418.9010563505249, "output_per_gpu": 200.9372404373607, "total_per_gpu": 1806.3059553023497, "tpot_error_pct": 0.23, "tpot_ms": 2.3871985635749446, "tpot_relative": 1.002272, "ttft_error_pct": 53.8, "ttft_ms": 38.335374000000186, "ttft_relative": 0.462011, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2271.8758422881365, "interactivity_tok_s": 419.8528979831518, "output_per_gpu": 197.56538698451467, "total_per_gpu": 1775.9950036886728, "tpot_ms": 2.381786584786486, "tpot_relative": 1.0, "ttft_ms": 82.97494126018137, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3320.5548472709065, "interactivity_tok_s": 316.9204550632329, "output_per_gpu": 615.0652468770251, "total_per_gpu": null, "tpot_error_pct": 9.78, "tpot_ms": 3.1553659097216586, "tpot_relative": 1.32479, "ttft_error_pct": 10.06, "ttft_ms": 92.61552162564986, "ttft_relative": 1.116187, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2991.989497525013, "interactivity_tok_s": 311.6619672063004, "output_per_gpu": 591.6982521022795, "total_per_gpu": 5339.928624879284, "tpot_error_pct": 8.25, "tpot_ms": 3.208604530619752, "tpot_relative": 1.347142, "ttft_error_pct": 45.13, "ttft_ms": 56.50234246249995, "ttft_relative": 0.680957, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 3302.58277823159, "interactivity_tok_s": 285.94109052827395, "output_per_gpu": 536.5008734398807, "total_per_gpu": 4841.786098193749, "tpot_ms": 3.497223844787427, "tpot_relative": 1.46832, "ttft_ms": 102.97266910347389, "ttft_relative": 1.241009, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4204.465651790166, "interactivity_tok_s": 250.50717016624478, "output_per_gpu": 978.7760073472475, "total_per_gpu": null, "tpot_error_pct": 11.12, "tpot_ms": 3.991901706192151, "tpot_relative": 1.676011, "ttft_error_pct": 8.56, "ttft_ms": 120.75020635559522, "ttft_relative": 1.455261, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3829.885919965573, "interactivity_tok_s": 246.7169194170434, "output_per_gpu": 940.1747251377216, "total_per_gpu": 8407.341750381876, "tpot_error_pct": 9.76, "tpot_ms": 4.053228300527001, "tpot_relative": 1.70176, "ttft_error_pct": 40.75, "ttft_ms": 78.24835713750005, "ttft_relative": 0.943036, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 4288.734025853046, "interactivity_tok_s": 222.6460444339005, "output_per_gpu": 839.8331330219106, "total_per_gpu": 7510.055284218397, "tpot_ms": 4.491433937407685, "tpot_relative": 1.885742, "ttft_ms": 132.0567641130765, "ttft_relative": 1.591526, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5974.882406693201, "interactivity_tok_s": 175.3458796094183, "output_per_gpu": 1380.4965661859237, "total_per_gpu": null, "tpot_error_pct": 8.66, "tpot_ms": 5.703013964328634, "tpot_relative": 2.394427, "ttft_error_pct": 33.45, "ttft_ms": 140.6991211850083, "ttft_relative": 1.695682, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5448.8335386530425, "interactivity_tok_s": 173.20515895393402, "output_per_gpu": 1327.8848679986347, "total_per_gpu": 11969.259377722274, "tpot_error_pct": 7.53, "tpot_ms": 5.773500085329225, "tpot_relative": 2.424021, "ttft_error_pct": 40.79, "ttft_ms": 125.16503802187503, "ttft_relative": 1.508468, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 5968.5346985406795, "interactivity_tok_s": 160.15692100935146, "output_per_gpu": 1211.2219746529802, "total_per_gpu": 10917.68595907622, "tpot_ms": 6.2438762789502595, "tpot_relative": 2.62151, "ttft_ms": 211.40834624566196, "ttft_relative": 2.547858, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 1.86, "tpot_shape_error_pct": 2.21, "ttft_mape_pct": 17.32, "ttft_shape_error_pct": 20.26}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.77, "tpot_shape_error_pct": 2.39, "ttft_mape_pct": 39.08, "ttft_shape_error_pct": 17.43}, "framework": "trt", "id": "76f4b2a543a7332d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 2712.394918507541, "interactivity_tok_s": 388.43675031697387, "output_per_gpu": 372.848017095478, "total_per_gpu": null, "tpot_error_pct": 1.16, "tpot_ms": 2.574421702333715, "tpot_relative": 1.011612, "ttft_error_pct": 0.53, "ttft_ms": 78.76151702015096, "ttft_relative": 0.994719, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2462.182304524999, "interactivity_tok_s": 379.86764455975504, "output_per_gpu": 364.8211890293986, "total_per_gpu": 3279.5249149931174, "tpot_error_pct": 3.44, "tpot_ms": 2.6324958556524156, "tpot_relative": 1.034432, "ttft_error_pct": 46.09, "ttft_ms": 42.68180460000008, "ttft_relative": 0.53905, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2418.5572394693736, "interactivity_tok_s": 392.9471978104637, "output_per_gpu": 371.596245577384, "total_per_gpu": 3340.4286328081885, "tpot_ms": 2.5448711826222143, "tpot_relative": 1.0, "ttft_ms": 79.17967028915882, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3153.258076800416, "interactivity_tok_s": 333.5797760047108, "output_per_gpu": 644.1853922354875, "total_per_gpu": null, "tpot_error_pct": 1.84, "tpot_ms": 2.9977836545638725, "tpot_relative": 1.177971, "ttft_error_pct": 10.23, "ttft_ms": 86.52539818157439, "ttft_relative": 1.092773, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2877.5174623250086, "interactivity_tok_s": 327.33450742805354, "output_per_gpu": 632.6768009756232, "total_per_gpu": 5627.952655945358, "tpot_error_pct": 3.78, "tpot_ms": 3.0549788589575906, "tpot_relative": 1.200445, "ttft_error_pct": 48.55, "ttft_ms": 49.594500162499756, "ttft_relative": 0.626354, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 2821.357327720034, "interactivity_tok_s": 339.7022335480304, "output_per_gpu": 645.2422323177334, "total_per_gpu": 5739.727977224528, "tpot_ms": 2.9437545627989232, "tpot_relative": 1.15674, "ttft_ms": 96.38779677916318, "ttft_relative": 1.21733, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3794.7360383454134, "interactivity_tok_s": 277.02753150107236, "output_per_gpu": 1076.5635430616478, "total_per_gpu": null, "tpot_error_pct": 0.76, "tpot_ms": 3.609749524105076, "tpot_relative": 1.418441, "ttft_error_pct": 7.76, "ttft_ms": 101.96227518592067, "ttft_relative": 1.287733, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3424.1967489500066, "interactivity_tok_s": 272.20527473836927, "output_per_gpu": 1035.9150765453007, "total_per_gpu": 9348.874279304222, "tpot_error_pct": 1.0, "tpot_ms": 3.6736980977358074, "tpot_relative": 1.443569, "ttft_error_pct": 42.76, "ttft_ms": 63.27398264375017, "ttft_relative": 0.799119, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 3438.876667687555, "interactivity_tok_s": 274.9290960005047, "output_per_gpu": 1032.1996102672344, "total_per_gpu": 9315.343126115029, "tpot_ms": 3.6373014517101683, "tpot_relative": 1.429267, "ttft_ms": 110.5373818998487, "ttft_relative": 1.396032, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4914.422048415397, "interactivity_tok_s": 213.96082462570953, "output_per_gpu": 1674.4057608999376, "total_per_gpu": null, "tpot_error_pct": 4.12, "tpot_ms": 4.673752785115411, "tpot_relative": 1.836538, "ttft_error_pct": 11.36, "ttft_ms": 133.17294924233101, "ttft_relative": 1.681908, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4474.21687098138, "interactivity_tok_s": 210.99644764229902, "output_per_gpu": 1611.7997926951841, "total_per_gpu": 14413.226954594065, "tpot_error_pct": 2.77, "tpot_ms": 4.739416284843307, "tpot_relative": 1.86234, "ttft_error_pct": 41.57, "ttft_ms": 87.78814110937503, "ttft_relative": 1.108721, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016885799", "measured": {"e2e_ms": 4662.701270074467, "interactivity_tok_s": 205.15069003356328, "output_per_gpu": 1546.4314424661802, "total_per_gpu": 13828.682353106948, "tpot_ms": 4.874465690738827, "tpot_relative": 1.915408, "ttft_ms": 150.2321477913938, "ttft_relative": 1.897358, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7180.1670171276055, "interactivity_tok_s": 145.63929524669902, "output_per_gpu": 2296.4755879773907, "total_per_gpu": null, "tpot_error_pct": 3.13, "tpot_ms": 6.866278762926556, "tpot_relative": 2.698085, "ttft_error_pct": 22.8, "ttft_ms": 155.96384265373823, "ttft_relative": 1.969746, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6556.404122503064, "interactivity_tok_s": 143.70861673858755, "output_per_gpu": 2208.887612339095, "total_per_gpu": 19910.422511382563, "tpot_error_pct": 4.52, "tpot_ms": 6.958524984058855, "tpot_relative": 2.734333, "ttft_error_pct": 30.29, "ttft_ms": 140.83497789218762, "ttft_relative": 1.778676, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19842069116", "measured": {"e2e_ms": 6340.347640985419, "interactivity_tok_s": 150.19999892147734, "output_per_gpu": 2280.951164550462, "total_per_gpu": 20559.987371171876, "tpot_ms": 6.657789661654973, "tpot_relative": 2.61616, "ttft_ms": 202.02173162142572, "ttft_relative": 2.551434, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10462.035620035156, "interactivity_tok_s": 99.29819179302449, "output_per_gpu": 3147.098909393454, "total_per_gpu": null, "tpot_error_pct": 0.17, "tpot_ms": 10.070676836536798, "tpot_relative": 3.957244, "ttft_error_pct": 51.25, "ttft_ms": 159.7332162580106, "ttft_relative": 2.017351, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9637.9348492507, "interactivity_tok_s": 98.01987299404455, "output_per_gpu": 3006.9323278429, "total_per_gpu": 27146.83115864525, "tpot_error_pct": 1.13, "tpot_ms": 10.202012810818044, "tpot_relative": 4.008852, "ttft_error_pct": 25.25, "ttft_ms": 244.9118860390625, "ttft_relative": 3.093116, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19842069116", "measured": {"e2e_ms": 9613.614035186583, "interactivity_tok_s": 99.13166131322173, "output_per_gpu": 3011.4414852058644, "total_per_gpu": 27187.54020037083, "tpot_ms": 10.087594485482759, "tpot_relative": 3.963892, "ttft_ms": 327.6332524194004, "ttft_relative": 4.137846, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.16, "tpot_shape_error_pct": 10.47, "ttft_mape_pct": 21.72, "ttft_shape_error_pct": 24.93}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.66, "tpot_shape_error_pct": 10.33, "ttft_mape_pct": 40.04, "ttft_shape_error_pct": 22.14}, "framework": "vllm", "id": "c99de178bf12dc6e", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 2907.6268852596045, "interactivity_tok_s": 363.22122478695223, "output_per_gpu": 175.91665649849404, "total_per_gpu": null, "tpot_error_pct": 7.13, "tpot_ms": 2.7531430757840516, "tpot_relative": 1.071299, "ttft_error_pct": 4.45, "ttft_ms": 91.16151873251944, "ttft_relative": 0.955508, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2533.5151368499937, "interactivity_tok_s": 369.89328939585954, "output_per_gpu": 177.15802421328954, "total_per_gpu": 1592.5449830481705, "tpot_error_pct": 5.2, "tpot_ms": 2.7034824060563065, "tpot_relative": 1.051975, "ttft_error_pct": 49.06, "ttft_ms": 48.59695777499984, "ttft_relative": 0.509368, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2457.0576793979853, "interactivity_tok_s": 389.11846243718867, "output_per_gpu": 182.76102363964213, "total_per_gpu": 1642.912606338644, "tpot_ms": 2.569911470498318, "tpot_relative": 1.0, "ttft_ms": 95.40636119199917, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3492.8445198822033, "interactivity_tok_s": 301.5746426502825, "output_per_gpu": 292.8844940497096, "total_per_gpu": null, "tpot_error_pct": 9.65, "tpot_ms": 3.315928657700967, "tpot_relative": 1.290289, "ttft_error_pct": 1.99, "ttft_ms": 100.64950305411415, "ttft_relative": 1.054956, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3074.038813862494, "interactivity_tok_s": 307.14390990245136, "output_per_gpu": 296.2051208444089, "total_per_gpu": 2634.881497140807, "tpot_error_pct": 11.29, "tpot_ms": 3.2558027939333036, "tpot_relative": 1.266893, "ttft_error_pct": 39.37, "ttft_ms": 59.83596283750025, "ttft_relative": 0.62717, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 3495.695502735907, "interactivity_tok_s": 272.46786955794306, "output_per_gpu": 260.1551263492137, "total_per_gpu": 2314.2001287815037, "tpot_ms": 3.6701575184715125, "tpot_relative": 1.428126, "ttft_ms": 98.68221982615069, "ttft_relative": 1.034336, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4242.7549244819475, "interactivity_tok_s": 247.52836847212447, "output_per_gpu": 482.23384013862704, "total_per_gpu": null, "tpot_error_pct": 9.25, "tpot_ms": 4.039940982007545, "tpot_relative": 1.572016, "ttft_error_pct": 7.88, "ttft_ms": 109.89529988822837, "ttft_relative": 1.151866, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3729.74602528126, "interactivity_tok_s": 250.46027197892684, "output_per_gpu": 476.162823735805, "total_per_gpu": 4297.250302052022, "tpot_error_pct": 10.31, "tpot_ms": 3.9926491818396563, "tpot_relative": 1.553614, "ttft_error_pct": 35.4, "ttft_ms": 77.06948574999981, "ttft_relative": 0.807802, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 4191.8921843287535, "interactivity_tok_s": 224.64084936675894, "output_per_gpu": 422.3040403415617, "total_per_gpu": 3811.1882626151055, "tpot_ms": 4.451550120198104, "tpot_relative": 1.73218, "ttft_ms": 119.29684613423888, "ttft_relative": 1.250408, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5840.874348475578, "interactivity_tok_s": 178.79453860561915, "output_per_gpu": 700.5800426211839, "total_per_gpu": null, "tpot_error_pct": 3.62, "tpot_ms": 5.593012000247819, "tpot_relative": 2.176344, "ttft_error_pct": 44.1, "ttft_ms": 119.22307222205865, "ttft_relative": 1.249634, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5240.296744928106, "interactivity_tok_s": 180.46964743682943, "output_per_gpu": 689.7981131899336, "total_per_gpu": 6168.394364682427, "tpot_error_pct": 4.51, "tpot_ms": 5.541097986297304, "tpot_relative": 2.156144, "ttft_error_pct": 47.34, "ttft_ms": 112.30609115937486, "ttft_relative": 1.177134, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 5584.536947218294, "interactivity_tok_s": 172.3282543738737, "output_per_gpu": 643.5179957203646, "total_per_gpu": 5754.542818356836, "tpot_ms": 5.802878951181483, "tpot_relative": 2.258007, "ttft_ms": 213.28636013786308, "ttft_relative": 2.235557, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8796.002175203668, "interactivity_tok_s": 118.03599243849928, "output_per_gpu": 930.4226894203221, "total_per_gpu": null, "tpot_error_pct": 6.17, "tpot_ms": 8.471992138508377, "tpot_relative": 3.296609, "ttft_error_pct": 50.19, "ttft_ms": 129.15421750959914, "ttft_relative": 1.353728, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8052.908948707853, "interactivity_tok_s": 117.15767556025708, "output_per_gpu": 900.675914558994, "total_per_gpu": 8118.492722092613, "tpot_error_pct": 6.97, "tpot_ms": 8.535505635613907, "tpot_relative": 3.321323, "ttft_error_pct": 29.03, "ttft_ms": 184.04258158437517, "ttft_relative": 1.929039, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 7616.1461198360485, "interactivity_tok_s": 125.32386143159957, "output_per_gpu": 948.8319548087458, "total_per_gpu": 8552.560577103297, "tpot_ms": 7.979326431349942, "tpot_relative": 3.104903, "ttft_ms": 259.3195749381266, "ttft_relative": 2.718053, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.85, "tpot_shape_error_pct": 8.52, "ttft_mape_pct": 22.93, "ttft_shape_error_pct": 36.16}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.28, "tpot_shape_error_pct": 9.85, "ttft_mape_pct": 29.61, "ttft_shape_error_pct": 5.04}, "framework": "vllm", "id": "1042fb081471eec2", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 3248.5655909119264, "interactivity_tok_s": 325.4842413937774, "output_per_gpu": 314.9082176028426, "total_per_gpu": null, "tpot_error_pct": 13.14, "tpot_ms": 3.07234536368899, "tpot_relative": 1.131353, "ttft_error_pct": 22.32, "ttft_ms": 105.55628385808937, "ttft_relative": 1.223242, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 2820.8156773750047, "interactivity_tok_s": 333.04337983453377, "output_per_gpu": 318.8132250103069, "total_per_gpu": 2865.9407569837003, "tpot_error_pct": 10.57, "tpot_ms": 3.002611853437324, "tpot_relative": 1.105675, "ttft_error_pct": 29.35, "ttft_ms": 60.964843775000006, "ttft_relative": 0.706493, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2582.1945203177165, "interactivity_tok_s": 368.237715733512, "output_per_gpu": 347.9177556054445, "total_per_gpu": 3127.5731294889097, "tpot_ms": 2.7156370933054688, "tpot_relative": 1.0, "ttft_ms": 86.29223005264066, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 3908.579611173945, "interactivity_tok_s": 269.7516932630951, "output_per_gpu": 523.4638164080998, "total_per_gpu": null, "tpot_error_pct": 22.32, "tpot_ms": 3.7071129671266854, "tpot_relative": 1.365099, "ttft_error_pct": 2.34, "ttft_ms": 116.2030458033461, "ttft_relative": 1.346622, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3438.8761477375147, "interactivity_tok_s": 275.22939503402023, "output_per_gpu": 529.970067565623, "total_per_gpu": 4714.328776917504, "tpot_error_pct": 19.89, "tpot_ms": 3.63333284177874, "tpot_relative": 1.33793, "ttft_error_pct": 34.17, "ttft_ms": 74.74647318749997, "ttft_relative": 0.866202, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2919.315094250487, "interactivity_tok_s": 329.9648859746646, "output_per_gpu": 623.8393087029662, "total_per_gpu": 5549.339076261229, "tpot_ms": 3.0306255074571244, "tpot_relative": 1.115991, "ttft_ms": 113.54737787332851, "ttft_relative": 1.315847, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4706.422462950925, "interactivity_tok_s": 223.3826492017558, "output_per_gpu": 869.4502102631725, "total_per_gpu": null, "tpot_error_pct": 20.21, "tpot_ms": 4.476623424305507, "tpot_relative": 1.648462, "ttft_error_pct": 8.17, "ttft_ms": 126.83669988639124, "ttft_relative": 1.469851, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4137.271630406258, "interactivity_tok_s": 226.41064389111452, "output_per_gpu": 859.4801164077785, "total_per_gpu": 7756.592925218131, "tpot_error_pct": 18.6, "tpot_ms": 4.416753483024943, "tpot_relative": 1.626415, "ttft_error_pct": 30.35, "ttft_ms": 96.1998548187499, "ttft_relative": 1.114815, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3545.6275580378133, "interactivity_tok_s": 268.5324533345012, "output_per_gpu": 1000.1705492903061, "total_per_gpu": 9026.288867578636, "tpot_ms": 3.723944676267252, "tpot_relative": 1.371297, "ttft_ms": 138.12731694961258, "ttft_relative": 1.600692, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6245.164134401615, "interactivity_tok_s": 167.50057069984896, "output_per_gpu": 1310.45394866698, "total_per_gpu": null, "tpot_error_pct": 20.8, "tpot_ms": 5.970128912527351, "tpot_relative": 2.198427, "ttft_error_pct": 32.28, "ttft_ms": 137.72225688613506, "ttft_relative": 1.595998, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5601.269518228152, "interactivity_tok_s": 169.49348390850298, "output_per_gpu": 1290.6840037766378, "total_per_gpu": 11541.707324573978, "tpot_error_pct": 19.38, "tpot_ms": 5.899931825932767, "tpot_relative": 2.172577, "ttft_error_pct": 31.29, "ttft_ms": 139.74778487499998, "ttft_relative": 1.619471, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 4778.686171537265, "interactivity_tok_s": 202.3352555601761, "output_per_gpu": 1508.6999219316076, "total_per_gpu": 13491.275082507076, "tpot_ms": 4.942292420722458, "tpot_relative": 1.819938, "ttft_ms": 203.3744823464076, "ttft_relative": 2.356811, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9356.253454549138, "interactivity_tok_s": 111.1086966339205, "output_per_gpu": 1749.4181917487128, "total_per_gpu": null, "tpot_error_pct": 27.79, "tpot_ms": 9.000195576902383, "tpot_relative": 3.314211, "ttft_error_pct": 49.51, "ttft_ms": 149.05337937799973, "ttft_relative": 1.727309, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8539.14288386724, "interactivity_tok_s": 110.94961175386341, "output_per_gpu": 1699.3999615455489, "total_per_gpu": 15318.013945656956, "tpot_error_pct": 27.97, "tpot_ms": 9.013100489422657, "tpot_relative": 3.318964, "ttft_error_pct": 22.87, "ttft_ms": 227.6881204203124, "ttft_relative": 2.63857, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 6790.874428785173, "interactivity_tok_s": 141.98135579649693, "output_per_gpu": 2131.025606018706, "total_per_gpu": 19208.591673650968, "tpot_ms": 7.043178270767526, "tpot_relative": 2.593564, "ttft_ms": 295.21673078124877, "ttft_relative": 3.421128, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 42.51, "tpot_shape_error_pct": 15.86, "ttft_mape_pct": 40.09, "ttft_shape_error_pct": 45.24}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 39.44, "tpot_shape_error_pct": 18.58, "ttft_mape_pct": 25.13, "ttft_shape_error_pct": 6.67}, "framework": "vllm", "id": "0447567c7b48c1ae", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 4990.402083641852, "interactivity_tok_s": 214.50653914388693, "output_per_gpu": 819.9740083896759, "total_per_gpu": null, "tpot_error_pct": 25.88, "tpot_ms": 4.661862542704206, "tpot_relative": 1.25875, "ttft_error_pct": 74.42, "ttft_ms": 221.31670245544933, "ttft_relative": 1.744163, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4267.999127349983, "interactivity_tok_s": 223.63326569613866, "output_per_gpu": 846.2354756374743, "total_per_gpu": 7607.152242685412, "tpot_error_pct": 20.74, "tpot_ms": 4.471606658728261, "tpot_relative": 1.207379, "ttft_error_pct": 24.39, "ttft_ms": 157.83538627499996, "ttft_relative": 1.243876, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3530.196865237667, "interactivity_tok_s": 270.0101626459166, "output_per_gpu": 1019.0510428451504, "total_per_gpu": 9160.661127035246, "tpot_ms": 3.7035643036568615, "tpot_relative": 1.0, "ttft_ms": 126.88993696938269, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7034.259046495642, "interactivity_tok_s": 150.6488249277, "output_per_gpu": 1163.4487649523144, "total_per_gpu": null, "tpot_error_pct": 41.3, "tpot_ms": 6.63795419897848, "tpot_relative": 1.792315, "ttft_error_pct": 45.8, "ttft_ms": 243.631900940657, "ttft_relative": 1.920025, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6118.796449662526, "interactivity_tok_s": 156.37768314496105, "output_per_gpu": 1195.6480433793377, "total_per_gpu": 10635.842140783476, "tpot_error_pct": 36.13, "tpot_ms": 6.3947743686227065, "tpot_relative": 1.726654, "ttft_error_pct": 16.2, "ttft_ms": 194.1637570124998, "ttft_relative": 1.530175, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 4516.036037489539, "interactivity_tok_s": 212.86913917796457, "output_per_gpu": 1615.0106681230718, "total_per_gpu": 14366.266575646941, "tpot_ms": 4.697721820371397, "tpot_relative": 1.268433, "ttft_ms": 167.09493961534463, "ttft_relative": 1.316849, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9762.566295623588, "interactivity_tok_s": 107.72679951057609, "output_per_gpu": 1676.6083327226704, "total_per_gpu": null, "tpot_error_pct": 40.32, "tpot_ms": 9.282741198505809, "tpot_relative": 2.506434, "ttft_error_pct": 26.04, "ttft_ms": 266.32204955214473, "ttft_relative": 2.098843, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8502.749467418746, "interactivity_tok_s": 110.91395171318321, "output_per_gpu": 1687.635830634209, "total_per_gpu": 15230.490961155947, "tpot_error_pct": 36.29, "tpot_ms": 9.015998299167446, "tpot_relative": 2.434411, "ttft_error_pct": 19.07, "ttft_ms": 251.5991718500001, "ttft_relative": 1.982814, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 6264.731483688229, "interactivity_tok_s": 151.1592740188786, "output_per_gpu": 2278.0494028868943, "total_per_gpu": 20558.825671944258, "tpot_ms": 6.615538520482097, "tpot_relative": 1.786263, "ttft_ms": 211.30235546734184, "ttft_relative": 1.665241, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14354.224218345944, "interactivity_tok_s": 72.73550323924388, "output_per_gpu": 2280.5830187716138, "total_per_gpu": null, "tpot_error_pct": 40.44, "tpot_ms": 13.748444094911516, "tpot_relative": 3.71222, "ttft_error_pct": 5.93, "ttft_ms": 289.56590925146355, "ttft_relative": 2.282024, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12791.61780345634, "interactivity_tok_s": 74.52120578563452, "output_per_gpu": 2276.047261690032, "total_per_gpu": 20353.13932338043, "tpot_error_pct": 37.08, "tpot_ms": 13.418999188990181, "tpot_relative": 3.623266, "ttft_error_pct": 19.37, "ttft_ms": 367.4632066843747, "ttft_relative": 2.895921, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 9368.443396216026, "interactivity_tok_s": 102.1521143109892, "output_per_gpu": 3093.7539946715183, "total_per_gpu": 27665.33329323701, "tpot_ms": 9.789322587642449, "tpot_relative": 2.643217, "ttft_ms": 307.8256377542857, "ttft_relative": 2.425926, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23367.61693209459, "interactivity_tok_s": 44.37601862886733, "output_per_gpu": 2801.8261421461657, "total_per_gpu": null, "tpot_error_pct": 52.48, "tpot_ms": 22.534693983328275, "tpot_relative": 6.084596, "ttft_error_pct": 32.96, "ttft_ms": 314.6249871497648, "ttft_relative": 2.479511, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21203.198392422048, "interactivity_tok_s": 44.76621892228421, "output_per_gpu": 2747.7516804089837, "total_per_gpu": 24767.62358016507, "tpot_error_pct": 51.15, "tpot_ms": 22.338272565213437, "tpot_relative": 6.031561, "ttft_error_pct": 28.27, "ttft_ms": 601.9916990937502, "ttft_relative": 4.744204, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 14096.499018684699, "interactivity_tok_s": 67.66538108531849, "output_per_gpu": 4120.740080293725, "total_per_gpu": 37143.436180238496, "tpot_ms": 14.778605898030955, "tpot_relative": 3.990374, "ttft_ms": 469.31756540725473, "ttft_relative": 3.698619, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 36540.16743178602, "interactivity_tok_s": 28.260912446650433, "output_per_gpu": 3583.5632183253974, "total_per_gpu": null, "tpot_error_pct": 54.65, "tpot_ms": 35.38456169409785, "tpot_relative": 9.554191, "ttft_error_pct": 55.37, "ttft_ms": 341.76081872392626, "ttft_relative": 2.693364, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 33802.53008642886, "interactivity_tok_s": 28.155235130538557, "output_per_gpu": 3445.855071231915, "total_per_gpu": 31109.461609666658, "tpot_error_pct": 55.23, "tpot_ms": 35.51737342499941, "tpot_relative": 9.590052, "ttft_error_pct": 43.45, "ttft_ms": 1098.5088763507824, "ttft_relative": 8.657179, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 21828.02706092516, "interactivity_tok_s": 43.70671588988891, "output_per_gpu": 5323.469645442717, "total_per_gpu": 48060.7196593208, "tpot_ms": 22.879778991386985, "tpot_relative": 6.177773, "ttft_ms": 765.7573929700447, "ttft_relative": 6.034816, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 35.69, "tpot_shape_error_pct": 11.91, "ttft_mape_pct": 32.27, "ttft_shape_error_pct": 43.79}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.82, "tpot_shape_error_pct": 13.84, "ttft_mape_pct": 9.29, "ttft_shape_error_pct": 8.93}, "framework": "vllm", "id": "82c7d3fd3f97a2cb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 4061.225919205565, "interactivity_tok_s": 261.2433986097984, "output_per_gpu": 503.78876740750917, "total_per_gpu": null, "tpot_error_pct": 23.44, "tpot_ms": 3.8278479200679527, "tpot_relative": 1.234366, "ttft_error_pct": 42.73, "ttft_ms": 145.33749697604955, "ttft_relative": 1.427336, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3514.4744422249946, "interactivity_tok_s": 268.76609967508705, "output_per_gpu": 512.4937521069315, "total_per_gpu": 4607.013187157691, "tpot_error_pct": 19.98, "tpot_ms": 3.7207073407282616, "tpot_relative": 1.199816, "ttft_error_pct": 7.12, "ttft_ms": 94.56938005000003, "ttft_relative": 0.928751, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 2951.9291701959446, "interactivity_tok_s": 322.46994034493446, "output_per_gpu": 609.0723211914043, "total_per_gpu": 5475.196925085778, "tpot_ms": 3.1010642385158014, "tpot_relative": 1.0, "ttft_ms": 101.82430826826021, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5114.7091899025245, "interactivity_tok_s": 206.46838116558735, "output_per_gpu": 800.0454860812888, "total_per_gpu": null, "tpot_error_pct": 33.6, "tpot_ms": 4.8433566164206105, "tpot_relative": 1.561837, "ttft_error_pct": 23.79, "ttft_ms": 159.95537130423997, "ttft_relative": 1.570896, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4487.870439999989, "interactivity_tok_s": 211.84172823798076, "output_per_gpu": 813.1687878997268, "total_per_gpu": 7233.512328150727, "tpot_error_pct": 30.21, "tpot_ms": 4.720505295711195, "tpot_relative": 1.522221, "ttft_error_pct": 10.15, "ttft_ms": 116.10353083750014, "ttft_relative": 1.140234, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 3484.8270688860794, "interactivity_tok_s": 275.8451583786923, "output_per_gpu": 1045.7600473456398, "total_per_gpu": 9302.519116972018, "tpot_ms": 3.6252222293028473, "tpot_relative": 1.169025, "ttft_ms": 129.21277695713798, "ttft_relative": 1.268978, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6650.907774199326, "interactivity_tok_s": 157.96325562762866, "output_per_gpu": 1230.5087181855015, "total_per_gpu": null, "tpot_error_pct": 32.79, "tpot_ms": 6.330586160855844, "tpot_relative": 2.041424, "ttft_error_pct": 2.23, "ttft_ms": 174.71813164379847, "ttft_relative": 1.715878, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5823.026218875005, "interactivity_tok_s": 161.2937335814468, "output_per_gpu": 1226.2691411734731, "total_per_gpu": 11066.772067507243, "tpot_error_pct": 30.05, "tpot_ms": 6.199868883902056, "tpot_relative": 1.999271, "ttft_error_pct": 12.27, "ttft_ms": 149.9333317687503, "ttft_relative": 1.472471, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 4533.188753557624, "interactivity_tok_s": 209.76504815536325, "output_per_gpu": 1569.203383773684, "total_per_gpu": 14161.667771535143, "tpot_ms": 4.767238435544068, "tpot_relative": 1.537291, "ttft_ms": 170.8992477862921, "ttft_relative": 1.678374, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9241.253184019159, "interactivity_tok_s": 113.02172017044867, "output_per_gpu": 1771.1883522794376, "total_per_gpu": null, "tpot_error_pct": 36.6, "tpot_ms": 8.847856841073508, "tpot_relative": 2.853168, "ttft_error_pct": 23.04, "ttft_ms": 189.89563560096119, "ttft_relative": 1.864934, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8263.42596884691, "interactivity_tok_s": 115.07300701167814, "output_per_gpu": 1755.159484519086, "total_per_gpu": 15695.194965610755, "tpot_error_pct": 34.16, "tpot_ms": 8.690135297311866, "tpot_relative": 2.802307, "ttft_error_pct": 11.45, "ttft_ms": 218.49828239374983, "ttft_relative": 2.145836, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 6243.61689426878, "interactivity_tok_s": 154.3858784767084, "output_per_gpu": 2313.8768261713176, "total_per_gpu": 20691.423334169685, "tpot_ms": 6.477276353684551, "tpot_relative": 2.088727, "ttft_ms": 246.73959481879137, "ttft_relative": 2.42319, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14091.272272434217, "interactivity_tok_s": 73.67385750588352, "output_per_gpu": 2323.140122985146, "total_per_gpu": null, "tpot_error_pct": 43.21, "tpot_ms": 13.57333569672446, "tpot_relative": 4.376993, "ttft_error_pct": 39.89, "ttft_ms": 205.749854685095, "ttft_relative": 2.020636, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12831.199148796897, "interactivity_tok_s": 73.92783065602723, "output_per_gpu": 2265.4513007748187, "total_per_gpu": 20420.275040441233, "tpot_error_pct": 42.72, "tpot_ms": 13.52670558741022, "tpot_relative": 4.361956, "ttft_error_pct": 4.19, "ttft_ms": 356.66529471406295, "ttft_relative": 3.502752, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 9082.974934839149, "interactivity_tok_s": 105.50931749072979, "output_per_gpu": 3191.6201505441513, "total_per_gpu": 28768.555420474328, "tpot_ms": 9.477835927503385, "tpot_relative": 3.056317, "ttft_ms": 342.31311592129714, "ttft_relative": 3.361802, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21906.780139088565, "interactivity_tok_s": 47.179477090257734, "output_per_gpu": 2988.6637645656297, "total_per_gpu": null, "tpot_error_pct": 44.51, "tpot_ms": 21.195656706557557, "tpot_relative": 6.834962, "ttft_error_pct": 61.93, "ttft_ms": 223.62332828018359, "ttft_relative": 2.196168, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20338.842717132786, "interactivity_tok_s": 46.764190181951676, "output_per_gpu": 2859.756527194521, "total_per_gpu": 25818.115984763648, "tpot_error_pct": 45.79, "tpot_ms": 21.383883610710814, "tpot_relative": 6.89566, "ttft_error_pct": 10.55, "ttft_ms": 649.4374837210948, "ttft_relative": 6.37802, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696226421", "measured": {"e2e_ms": 14092.412710078497, "interactivity_tok_s": 68.17696246973699, "output_per_gpu": 4117.970190027403, "total_per_gpu": 37177.371911526694, "tpot_ms": 14.66771125868051, "tpot_relative": 4.729896, "ttft_ms": 587.4652929407603, "ttft_relative": 5.769401, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 6, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}, {"aic": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.34, "tpot_shape_error_pct": 7.07, "ttft_mape_pct": 45.43, "ttft_shape_error_pct": 32.31}, "aisimulate": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.63, "tpot_shape_error_pct": 6.38, "ttft_mape_pct": 50.66, "ttft_shape_error_pct": 10.86}, "gpu": "h100", "precisions": ["fp4"], "rows": 15, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.9, "tpot_shape_error_pct": 6.62, "ttft_mape_pct": 50.34, "ttft_shape_error_pct": 35.37}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.05, "tpot_shape_error_pct": 5.56, "ttft_mape_pct": 51.01, "ttft_shape_error_pct": 2.72}, "framework": "vllm", "id": "47e53888324ca2dd", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 5913.825924652376, "interactivity_tok_s": 178.80273023901424, "output_per_gpu": 345.9689253738505, "total_per_gpu": null, "tpot_error_pct": 1.09, "tpot_ms": 5.592755763087352, "tpot_relative": 0.989102, "ttft_error_pct": 30.74, "ttft_ms": 192.43677901401514, "ttft_relative": 0.692562, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5117.552770024992, "interactivity_tok_s": 184.51866377643043, "output_per_gpu": 352.2180842213022, "total_per_gpu": 3166.230519088139, "tpot_error_pct": 4.15, "tpot_ms": 5.419505970472649, "tpot_relative": 0.958462, "ttft_error_pct": 50.98, "ttft_ms": 136.20374107500055, "ttft_relative": 0.490185, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5472.94934723177, "interactivity_tok_s": 176.8541127485591, "output_per_gpu": 328.9741180876137, "total_per_gpu": 2957.2811259306463, "tpot_ms": 5.65437797548843, "tpot_relative": 1.0, "ttft_ms": 277.8621904843021, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8065.231241059916, "interactivity_tok_s": 130.266436704287, "output_per_gpu": 507.3630101475228, "total_per_gpu": null, "tpot_error_pct": 2.25, "tpot_ms": 7.676574452328521, "tpot_relative": 1.357634, "ttft_error_pct": 40.63, "ttft_ms": 212.09557632783904, "ttft_relative": 0.763312, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7070.081324312401, "interactivity_tok_s": 134.17086601667603, "output_per_gpu": 516.7505054473058, "total_per_gpu": 4596.734659953691, "tpot_error_pct": 0.73, "tpot_ms": 7.453182868147475, "tpot_relative": 1.318126, "ttft_error_pct": 53.09, "ttft_ms": 167.59091854999997, "ttft_relative": 0.603144, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 7313.1258495152, "interactivity_tok_s": 133.1922530916249, "output_per_gpu": 499.1324317853124, "total_per_gpu": 4440.013749204702, "tpot_ms": 7.507944169336075, "tpot_relative": 1.327811, "ttft_ms": 357.2533421742264, "ttft_relative": 1.285721, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10982.099658068808, "interactivity_tok_s": 95.16347216673985, "output_per_gpu": 745.2126874469785, "total_per_gpu": null, "tpot_error_pct": 2.87, "tpot_ms": 10.508233645025674, "tpot_relative": 1.858424, "ttft_error_pct": 46.71, "ttft_ms": 232.17663920754214, "ttft_relative": 0.835582, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9623.770413624987, "interactivity_tok_s": 97.26211873987636, "output_per_gpu": 743.5581826461851, "total_per_gpu": 6710.426487941092, "tpot_error_pct": 4.96, "tpot_ms": 10.281495128380454, "tpot_relative": 1.818325, "ttft_error_pct": 50.19, "ttft_ms": 216.99016187500038, "ttft_relative": 0.780927, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 10341.767730933498, "interactivity_tok_s": 92.43543294935505, "output_per_gpu": 692.6718285399187, "total_per_gpu": 6251.189878837801, "tpot_ms": 10.818362267507261, "tpot_relative": 1.913272, "ttft_ms": 435.6616013668827, "ttft_relative": 1.567905, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15141.785198153088, "interactivity_tok_s": 68.70976440550392, "output_per_gpu": 1080.9821818101395, "total_per_gpu": null, "tpot_error_pct": 11.95, "tpot_ms": 14.553972185063934, "tpot_relative": 2.57393, "ttft_error_pct": 59.35, "ttft_ms": 253.0716528326827, "ttft_relative": 0.910781, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13531.05255013131, "interactivity_tok_s": 70.0402215338673, "output_per_gpu": 1072.942437158215, "total_per_gpu": 9594.592905436126, "tpot_error_pct": 13.63, "tpot_ms": 14.277510523242125, "tpot_relative": 2.525036, "ttft_error_pct": 49.19, "ttft_ms": 316.3365431812499, "ttft_relative": 1.138466, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 15931.96927531535, "interactivity_tok_s": 60.496524896042374, "output_per_gpu": 912.4854921693673, "total_per_gpu": 8159.735812733643, "tpot_ms": 16.52987509147685, "tpot_relative": 2.923376, "ttft_ms": 622.5760209192231, "ttft_relative": 2.240593, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24312.683061073818, "interactivity_tok_s": 42.56035048806754, "output_per_gpu": 1346.4577281646245, "total_per_gpu": null, "tpot_error_pct": 11.32, "tpot_ms": 23.496047107985298, "tpot_relative": 4.155373, "ttft_error_pct": 74.26, "ttft_ms": 276.2268696048611, "ttft_relative": 0.994115, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22072.31935643916, "interactivity_tok_s": 42.775277929830054, "output_per_gpu": 1317.3202166698482, "total_per_gpu": 11874.031956251592, "tpot_error_pct": 11.77, "tpot_ms": 23.37798953966897, "tpot_relative": 4.134494, "ttft_error_pct": 51.62, "ttft_ms": 519.2564843546877, "ttft_relative": 1.868755, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 25521.54412102973, "interactivity_tok_s": 37.74138357178473, "output_per_gpu": 1140.9360301350728, "total_per_gpu": 10284.144060364037, "tpot_ms": 26.496113956659368, "tpot_relative": 4.685947, "ttft_ms": 1073.256870278783, "ttft_relative": 3.862551, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.42, "tpot_shape_error_pct": 3.49, "ttft_mape_pct": 45.62, "ttft_shape_error_pct": 38.1}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.98, "tpot_shape_error_pct": 2.85, "ttft_mape_pct": 51.53, "ttft_shape_error_pct": 7.65}, "framework": "vllm", "id": "774ea1c1be6f2e30", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 4328.58464634747, "interactivity_tok_s": 244.1335795720995, "output_per_gpu": 236.33591198527768, "total_per_gpu": null, "tpot_error_pct": 7.38, "tpot_ms": 4.096118206076898, "tpot_relative": 0.926242, "ttft_error_pct": 21.77, "ttft_ms": 138.25572153080302, "ttft_relative": 0.782312, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3767.8841670749994, "interactivity_tok_s": 249.90504960544106, "output_per_gpu": 238.5444029548486, "total_per_gpu": 2144.3719179357813, "tpot_error_pct": 9.51, "tpot_ms": 4.001519783529126, "tpot_relative": 0.904851, "ttft_error_pct": 49.16, "ttft_ms": 89.84983942500037, "ttft_relative": 0.50841, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 4241.741585545242, "interactivity_tok_s": 226.1267924174651, "output_per_gpu": 212.44151840206214, "total_per_gpu": 1909.7225532105492, "tpot_ms": 4.42229772646244, "tpot_relative": 1.0, "ttft_ms": 176.7270419630222, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5537.099194932233, "interactivity_tok_s": 189.97577728437872, "output_per_gpu": 369.50755765267456, "total_per_gpu": null, "tpot_error_pct": 4.15, "tpot_ms": 5.263828969643214, "tpot_relative": 1.190293, "ttft_error_pct": 27.26, "ttft_ms": 152.20215898722597, "ttft_relative": 0.861227, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4868.658000224983, "interactivity_tok_s": 194.6017321156114, "output_per_gpu": 374.5814031983626, "total_per_gpu": 3332.0747651045745, "tpot_error_pct": 6.43, "tpot_ms": 5.138700406869491, "tpot_relative": 1.161998, "ttft_error_pct": 47.31, "ttft_ms": 110.24362756249998, "ttft_relative": 0.623807, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5295.437173813116, "interactivity_tok_s": 182.08638173341288, "output_per_gpu": 344.4308777824643, "total_per_gpu": 3063.871901761256, "tpot_ms": 5.491898902489421, "tpot_relative": 1.241865, "ttft_ms": 209.24990483326837, "ttft_relative": 1.184029, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7273.256018082968, "interactivity_tok_s": 143.9485280586643, "output_per_gpu": 562.6090969197782, "total_per_gpu": null, "tpot_error_pct": 6.76, "tpot_ms": 6.946927582284575, "tpot_relative": 1.570886, "ttft_error_pct": 45.93, "ttft_ms": 166.5491014058479, "ttft_relative": 0.942409, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6372.5217189812465, "interactivity_tok_s": 146.85165245147581, "output_per_gpu": 559.6055448625842, "total_per_gpu": 5050.299974751893, "tpot_error_pct": 8.61, "tpot_ms": 6.809593105058385, "tpot_relative": 1.539831, "ttft_error_pct": 53.79, "ttft_ms": 142.3273386625001, "ttft_relative": 0.805351, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 7127.496919991972, "interactivity_tok_s": 134.214013331171, "output_per_gpu": 500.51545323592654, "total_per_gpu": 4517.026687898604, "tpot_ms": 7.450786808174162, "tpot_relative": 1.684823, "ttft_ms": 308.0236949157552, "ttft_relative": 1.742935, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10058.506978488102, "interactivity_tok_s": 103.5719323557051, "output_per_gpu": 813.6396403067506, "total_per_gpu": null, "tpot_error_pct": 9.38, "tpot_ms": 9.655125450065203, "tpot_relative": 2.183283, "ttft_error_pct": 61.13, "ttft_ms": 181.31364307139836, "ttft_relative": 1.025953, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9011.809850671936, "interactivity_tok_s": 105.12271253950371, "output_per_gpu": 804.1834876220123, "total_per_gpu": 7191.264803956369, "tpot_error_pct": 10.72, "tpot_ms": 9.512692127538218, "tpot_relative": 2.151075, "ttft_error_pct": 55.55, "ttft_ms": 207.36081601562492, "ttft_relative": 1.173339, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 10334.076415480013, "interactivity_tok_s": 93.85464020482725, "output_per_gpu": 701.1785159926559, "total_per_gpu": 6270.161550144124, "tpot_ms": 10.654774210604952, "tpot_relative": 2.40933, "ttft_ms": 466.497043997515, "ttft_relative": 2.639647, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14922.357502654295, "interactivity_tok_s": 69.47207174111253, "output_per_gpu": 1096.8776211861004, "total_per_gpu": null, "tpot_error_pct": 14.45, "tpot_ms": 14.39427348196117, "tpot_relative": 3.254931, "ttft_error_pct": 71.98, "ttft_ms": 197.0157306080174, "ttft_relative": 1.114802, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 13582.483827826509, "interactivity_tok_s": 69.62022755832565, "output_per_gpu": 1068.8856064310705, "total_per_gpu": 9634.697538025266, "tpot_error_pct": 14.63, "tpot_ms": 14.363641646563583, "tpot_relative": 3.248004, "ttft_error_pct": 51.85, "ttft_ms": 338.61099063437496, "ttft_relative": 1.916011, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 16224.091354128905, "interactivity_tok_s": 59.43574427604359, "output_per_gpu": 895.8205752498997, "total_per_gpu": 8074.727771562683, "tpot_ms": 16.824892363686004, "tpot_relative": 3.804559, "ttft_ms": 703.187489947959, "ttft_relative": 3.978947, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.7, "tpot_shape_error_pct": 11.1, "ttft_mape_pct": 40.34, "ttft_shape_error_pct": 23.47}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.87, "tpot_shape_error_pct": 10.73, "ttft_mape_pct": 49.44, "ttft_shape_error_pct": 22.2}, "framework": "vllm", "id": "392ad9f63f58e7ff", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 3826.723194002638, "interactivity_tok_s": 275.3135744118993, "output_per_gpu": 133.66527288977656, "total_per_gpu": null, "tpot_error_pct": 8.73, "tpot_ms": 3.632221920535928, "tpot_relative": 0.912658, "ttft_error_pct": 27.81, "ttft_ms": 110.96016929438376, "ttft_relative": 0.721913, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3343.547063150005, "interactivity_tok_s": 280.4406136708775, "output_per_gpu": 134.20980697079992, "total_per_gpu": 1206.466123769162, "tpot_error_pct": 10.4, "tpot_ms": 3.565817329060586, "tpot_relative": 0.895973, "ttft_error_pct": 57.07, "ttft_ms": 65.99206947500002, "ttft_relative": 0.429348, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 3811.1384132003877, "interactivity_tok_s": 251.26718055109563, "output_per_gpu": 117.84983710088686, "total_per_gpu": 1059.399751501533, "tpot_ms": 3.9798273606872754, "tpot_relative": 1.0, "ttft_ms": 153.70296967448667, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4365.783745026477, "interactivity_tok_s": 241.06519207595244, "output_per_gpu": 234.3221881215273, "total_per_gpu": null, "tpot_error_pct": 8.96, "tpot_ms": 4.148255463131856, "tpot_relative": 1.04232, "ttft_error_pct": 25.24, "ttft_ms": 122.11840624258812, "ttft_relative": 0.794509, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3850.1075164750036, "interactivity_tok_s": 245.61719628054664, "output_per_gpu": 236.3193227113297, "total_per_gpu": 2102.1696351968503, "tpot_error_pct": 10.65, "tpot_ms": 4.071376170493328, "tpot_relative": 1.023003, "ttft_error_pct": 50.55, "ttft_ms": 80.7836680625001, "ttft_relative": 0.525583, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 4382.676460291259, "interactivity_tok_s": 219.4574144901706, "output_per_gpu": 207.7334442736315, "total_per_gpu": 1847.8850301222074, "tpot_ms": 4.556692706524115, "tpot_relative": 1.144947, "ttft_ms": 163.35672289133072, "ttft_relative": 1.062808, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5622.508363000169, "interactivity_tok_s": 186.36915730287265, "output_per_gpu": 363.894523210323, "total_per_gpu": null, "tpot_error_pct": 8.38, "tpot_ms": 5.365694702234865, "tpot_relative": 1.348223, "ttft_error_pct": 36.57, "ttft_ms": 133.40268261390167, "ttft_relative": 0.867925, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4935.238600268737, "interactivity_tok_s": 189.3731373883102, "output_per_gpu": 359.9757614037944, "total_per_gpu": 3248.691145787875, "tpot_error_pct": 9.83, "tpot_ms": 5.280579990336734, "tpot_relative": 1.326836, "ttft_error_pct": 50.44, "ttft_ms": 104.21796051874988, "ttft_relative": 0.678048, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "25956497716", "measured": {"e2e_ms": 5570.12688726536, "interactivity_tok_s": 170.75075302616872, "output_per_gpu": 319.43254438864045, "total_per_gpu": 2882.798760074878, "tpot_ms": 5.856489545593647, "tpot_relative": 1.471544, "ttft_ms": 210.2982644137228, "ttft_relative": 1.368212, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7333.555056135787, "interactivity_tok_s": 142.3123897773375, "output_per_gpu": 557.9831294204758, "total_per_gpu": null, "tpot_error_pct": 30.35, "tpot_ms": 7.026795077818619, "tpot_relative": 1.765603, "ttft_error_pct": 48.04, "ttft_ms": 145.14369152734008, "ttft_relative": 0.944313, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6560.812972084386, "interactivity_tok_s": 144.4126842188794, "output_per_gpu": 550.4060187497769, "total_per_gpu": 4921.905872781909, "tpot_error_pct": 31.36, "tpot_ms": 6.9245994935205815, "tpot_relative": 1.739925, "ttft_error_pct": 45.73, "ttft_ms": 151.58064370312488, "ttft_relative": 0.986192, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 9617.295203893445, "interactivity_tok_s": 99.12728878111258, "output_per_gpu": 375.3112613362415, "total_per_gpu": 3356.152800596136, "tpot_ms": 10.088039452063951, "tpot_relative": 2.534793, "ttft_ms": 279.31090809288435, "ttft_relative": 1.817212, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10893.858454524492, "interactivity_tok_s": 95.28282310136143, "output_per_gpu": 751.2489752059317, "total_per_gpu": null, "tpot_error_pct": 27.08, "tpot_ms": 10.495071067911208, "tpot_relative": 2.637067, "ttft_error_pct": 64.04, "ttft_ms": 157.4007520513251, "ttft_relative": 1.024058, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9923.802362549974, "interactivity_tok_s": 95.29058085064472, "output_per_gpu": 730.4775845703319, "total_per_gpu": 6584.362763702607, "tpot_error_pct": 27.08, "tpot_ms": 10.4942166484153, "tpot_relative": 2.636852, "ttft_error_pct": 43.41, "ttft_ms": 247.70958476874972, "ttft_relative": 1.611612, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 13709.871438669506, "interactivity_tok_s": 69.48433028234061, "output_per_gpu": 528.7977034607527, "total_per_gpu": 4766.465093171108, "tpot_ms": 14.391734020269448, "tpot_relative": 3.61617, "ttft_ms": 437.73427094420185, "ttft_relative": 2.847923, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 37, "status_counts": {"failed": 0, "success": 37, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.89, "tpot_shape_error_pct": 20.23, "ttft_mape_pct": 58.99, "ttft_shape_error_pct": 70.72}, "aisimulate": {"coverage_pct": 100.0, "points": 37, "status_counts": {"failed": 0, "success": 37, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.64, "tpot_shape_error_pct": 18.73, "ttft_mape_pct": 66.85, "ttft_shape_error_pct": 76.62}, "gpu": "h200", "precisions": ["fp4"], "rows": 37, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.17, "tpot_shape_error_pct": 10.2, "ttft_mape_pct": 48.94, "ttft_shape_error_pct": 7.45}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.72, "tpot_shape_error_pct": 6.03, "ttft_mape_pct": 66.8, "ttft_shape_error_pct": 32.74}, "framework": "trt", "id": "2964c1c3bb9b7ad9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 8190.1517862176215, "interactivity_tok_s": 133.76257494080693, "output_per_gpu": 485.6110419856342, "total_per_gpu": null, "tpot_error_pct": 23.42, "tpot_ms": 7.475932639922067, "tpot_relative": 0.76579, "ttft_error_pct": 50.22, "ttft_ms": 542.2726955773468, "ttft_relative": 0.497793, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7499.82997502501, "interactivity_tok_s": 127.41240553602799, "output_per_gpu": 481.38445441880515, "total_per_gpu": 4327.359154102263, "tpot_error_pct": 19.6, "tpot_ms": 7.848529315438074, "tpot_relative": 0.803957, "ttft_error_pct": 73.69, "ttft_ms": 286.6050952750003, "ttft_relative": 0.263096, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 10073.385546426289, "interactivity_tok_s": 102.43403872805644, "output_per_gpu": 359.8686318069966, "total_per_gpu": 3235.0043792013, "tpot_ms": 9.76237989263331, "tpot_relative": 1.0, "ttft_ms": 1089.3548449734226, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12329.035003926474, "interactivity_tok_s": 87.1950628909959, "output_per_gpu": 654.4579874605606, "total_per_gpu": null, "tpot_error_pct": 18.55, "tpot_ms": 11.468539236563403, "tpot_relative": 1.174769, "ttft_error_pct": 50.89, "ttft_ms": 596.7193649221113, "ttft_relative": 0.547773, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11248.29726029999, "interactivity_tok_s": 84.89938532293, "output_per_gpu": 650.6532199599031, "total_per_gpu": 5787.8612140131645, "tpot_error_pct": 16.35, "tpot_ms": 11.77864829287422, "tpot_relative": 1.206535, "ttft_error_pct": 72.33, "ttft_ms": 336.27205063749943, "ttft_relative": 0.308689, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 14302.650264836848, "interactivity_tok_s": 71.01890731819513, "output_per_gpu": 512.6492463976177, "total_per_gpu": 4560.25206453401, "tpot_ms": 14.080757332967282, "tpot_relative": 1.442349, "ttft_ms": 1215.1382524287328, "ttft_relative": 1.115466, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17545.35477531544, "interactivity_tok_s": 60.74616859491063, "output_per_gpu": 929.708626310215, "total_per_gpu": null, "tpot_error_pct": 17.36, "tpot_ms": 16.46194357817953, "tpot_relative": 1.686263, "ttft_error_pct": 44.41, "ttft_ms": 704.7864948377825, "ttft_relative": 0.646976, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15718.981830687491, "interactivity_tok_s": 59.87823498521727, "output_per_gpu": 913.0503146167874, "total_per_gpu": 8240.050555590336, "tpot_error_pct": 16.17, "tpot_ms": 16.70055906368783, "tpot_relative": 1.710706, "ttft_error_pct": 65.73, "ttft_ms": 434.50593741874917, "ttft_relative": 0.398865, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 19533.372008951847, "interactivity_tok_s": 50.19882341140569, "output_per_gpu": 737.1439270728388, "total_per_gpu": 6652.539436861072, "tpot_ms": 19.92078562886774, "tpot_relative": 2.040567, "ttft_ms": 1267.726926296018, "ttft_relative": 1.163741, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25039.55932403563, "interactivity_tok_s": 42.42359034126848, "output_per_gpu": 1316.195607855918, "total_per_gpu": null, "tpot_error_pct": 17.13, "tpot_ms": 23.571790882282492, "tpot_relative": 2.414554, "ttft_error_pct": 45.43, "ttft_ms": 925.6172514606383, "ttft_relative": 0.849693, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22484.295535756184, "interactivity_tok_s": 42.324605586328545, "output_per_gpu": 1294.30095150752, "total_per_gpu": 11574.051222844952, "tpot_error_pct": 16.94, "tpot_ms": 23.626918340923993, "tpot_relative": 2.420201, "ttft_error_pct": 64.2, "ttft_ms": 607.2783494562492, "ttft_relative": 0.557466, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 28078.00020739669, "interactivity_tok_s": 35.15467957378138, "output_per_gpu": 1040.2518717177136, "total_per_gpu": 9302.263460362758, "tpot_ms": 28.445715111731737, "tpot_relative": 2.913809, "ttft_ms": 1696.2571826588828, "ttft_relative": 1.557121, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 41672.01488272345, "interactivity_tok_s": 25.205019556472113, "output_per_gpu": 1584.455520003061, "total_per_gpu": null, "tpot_error_pct": 9.4, "tpot_ms": 39.67463694124456, "tpot_relative": 4.064033, "ttft_error_pct": 53.77, "ttft_ms": 1084.8612918302697, "ttft_relative": 0.995875, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37510.50987509514, "interactivity_tok_s": 25.248861292479614, "output_per_gpu": 1552.88659577823, "total_per_gpu": 13997.374996125725, "tpot_error_pct": 9.56, "tpot_ms": 39.605746509362405, "tpot_relative": 4.056977, "ttft_error_pct": 58.06, "ttft_ms": 984.2703806015619, "ttft_relative": 0.903535, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 42833.98088179529, "interactivity_tok_s": 22.835938072106917, "output_per_gpu": 1363.6747425687536, "total_per_gpu": 12291.861360883317, "tpot_ms": 43.790624972024055, "tpot_relative": 4.485651, "ttft_ms": 2346.8190781743033, "ttft_relative": 2.15432, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.15, "tpot_shape_error_pct": 35.89, "ttft_mape_pct": 46.17, "ttft_shape_error_pct": 389.29}, "aisimulate": {"points": 2, "status_counts": {"failed": 0, "success": 2, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.25, "tpot_shape_error_pct": 34.62, "ttft_mape_pct": 69.43, "ttft_shape_error_pct": 422.4}, "framework": "trt", "id": "4e9bfc6c4c2254d5", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5212.543550699682, "interactivity_tok_s": 203.7224084691116, "output_per_gpu": 96.6692004589778, "total_per_gpu": null, "tpot_error_pct": 33.15, "tpot_ms": 4.908640181090437, "tpot_relative": 0.668526, "ttft_error_pct": 81.73, "ttft_ms": 191.00464544416468, "ttft_relative": 0.18269, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4747.743313099987, "interactivity_tok_s": 197.8639480423406, "output_per_gpu": 94.6588669554732, "total_per_gpu": 850.9267606725468, "tpot_error_pct": 31.17, "tpot_ms": 5.053977795823681, "tpot_relative": 0.688321, "ttft_error_pct": 90.18, "ttft_ms": 102.70192887499998, "ttft_relative": 0.098231, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 7800.690950383432, "interactivity_tok_s": 136.19382568432388, "output_per_gpu": 58.028002461265224, "total_per_gpu": 521.6371350175765, "tpot_ms": 7.342476760421171, "tpot_relative": 1.0, "ttft_ms": 1045.5109692411497, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6215.278339799284, "interactivity_tok_s": 170.34854705765522, "output_per_gpu": 163.06637474954, "total_per_gpu": null, "tpot_error_pct": 9.16, "tpot_ms": 5.8703171660251705, "tpot_relative": 0.799501, "ttft_error_pct": 10.61, "ttft_ms": 209.94387895553544, "ttft_relative": 0.200805, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5664.188888849996, "interactivity_tok_s": 167.01236367417525, "output_per_gpu": 160.8178698000211, "total_per_gpu": 1430.5493042716696, "tpot_error_pct": 7.34, "tpot_ms": 5.987580667685789, "tpot_relative": 0.815472, "ttft_error_pct": 48.68, "ttft_ms": 120.52360984999987, "ttft_relative": 0.115277, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 6219.374778994825, "interactivity_tok_s": 154.75124436740197, "output_per_gpu": 146.3981286568349, "total_per_gpu": 1302.2790399918733, "tpot_ms": 6.461983579439623, "tpot_relative": 0.880082, "ttft_ms": 234.8648586892523, "ttft_relative": 0.224641, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 2, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 45.42, "tpot_shape_error_pct": 90.16, "ttft_mape_pct": 80.39, "ttft_shape_error_pct": 207.66}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.69, "tpot_shape_error_pct": 86.28, "ttft_mape_pct": 86.5, "ttft_shape_error_pct": 302.2}, "framework": "trt", "id": "b74798202c37d38d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 5827.948497279177, "interactivity_tok_s": 183.35591993939858, "output_per_gpu": 172.44686040284435, "total_per_gpu": null, "tpot_error_pct": 68.29, "tpot_ms": 5.45387353912823, "tpot_relative": 0.317113, "ttft_error_pct": 92.63, "ttft_ms": 248.6358667509978, "ttft_relative": 0.07369, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5305.804709774998, "interactivity_tok_s": 177.68396282258922, "output_per_gpu": 169.6204123035929, "total_per_gpu": 1524.7863473089317, "tpot_error_pct": 67.28, "tpot_ms": 5.627969931076236, "tpot_relative": 0.327236, "ttft_error_pct": 96.05, "ttft_ms": 133.2509786500004, "ttft_relative": 0.039492, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 18167.46389610464, "interactivity_tok_s": 58.14459547141532, "output_per_gpu": 45.30972594585339, "total_per_gpu": 445.5112475464269, "tpot_ms": 17.198503006038003, "tpot_relative": 1.0, "ttft_ms": 3374.0832482781743, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7390.179707467871, "interactivity_tok_s": 143.74596158423554, "output_per_gpu": 273.9902519486481, "total_per_gpu": null, "tpot_error_pct": 44.18, "tpot_ms": 6.956717176461317, "tpot_relative": 0.404496, "ttft_error_pct": 80.45, "ttft_ms": 273.4580359479437, "ttft_relative": 0.081047, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6724.455363325008, "interactivity_tok_s": 140.99754970169207, "output_per_gpu": 271.266719166369, "total_per_gpu": 2413.0428842147035, "tpot_error_pct": 43.1, "tpot_ms": 7.0923218319445684, "tpot_relative": 0.41238, "ttft_error_pct": 88.78, "ttft_ms": 156.99141619999995, "ttft_relative": 0.046529, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 12970.968483099932, "interactivity_tok_s": 80.23446427181891, "output_per_gpu": 136.68844121151858, "total_per_gpu": 1215.906880996948, "tpot_ms": 12.463472013874146, "tpot_relative": 0.724684, "ttft_ms": 1399.06541993696, "ttft_relative": 0.414651, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9282.853010136394, "interactivity_tok_s": 114.18353876412289, "output_per_gpu": 439.6182382035004, "total_per_gpu": null, "tpot_error_pct": 38.85, "tpot_ms": 8.757829813505532, "tpot_relative": 0.50922, "ttft_error_pct": 73.64, "ttft_ms": 323.5931109202336, "ttft_relative": 0.095905, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8318.082566949994, "interactivity_tok_s": 112.76833713052334, "output_per_gpu": 428.28347805140606, "total_per_gpu": 3865.151191310662, "tpot_error_pct": 38.08, "tpot_ms": 8.867737393720308, "tpot_relative": 0.515611, "ttft_error_pct": 83.4, "ttft_ms": 203.7715493812503, "ttft_relative": 0.060393, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 14363.246830360731, "interactivity_tok_s": 69.82305169505216, "output_per_gpu": 250.4890854487848, "total_per_gpu": 2260.601299489027, "tpot_ms": 14.321917700868159, "tpot_relative": 0.832742, "ttft_ms": 1227.4979356850963, "ttft_relative": 0.363802, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12213.410891288157, "interactivity_tok_s": 86.7710851909281, "output_per_gpu": 674.4079931002167, "total_per_gpu": null, "tpot_error_pct": 40.04, "tpot_ms": 11.524576393157172, "tpot_relative": 0.670092, "ttft_error_pct": 76.07, "ttft_ms": 423.7692410883709, "ttft_relative": 0.125595, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11044.131386403114, "interactivity_tok_s": 86.05971136781427, "output_per_gpu": 655.1796521028531, "total_per_gpu": 5858.825062881905, "tpot_error_pct": 39.54, "tpot_ms": 11.619839110615388, "tpot_relative": 0.675631, "ttft_error_pct": 83.85, "ttft_ms": 285.87423760624966, "ttft_relative": 0.084726, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 19609.323627103004, "interactivity_tok_s": 52.03148925084096, "output_per_gpu": 372.2396359760812, "total_per_gpu": 3328.68534859861, "tpot_ms": 19.219130845535762, "tpot_relative": 1.117489, "ttft_ms": 1770.5642221233575, "ttft_relative": 0.524754, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18068.03831841704, "interactivity_tok_s": 58.21434816005124, "output_per_gpu": 913.9069415985253, "total_per_gpu": null, "tpot_error_pct": 35.72, "tpot_ms": 17.17789568390694, "tpot_relative": 0.998802, "ttft_error_pct": 79.16, "ttft_ms": 495.0510337802396, "ttft_relative": 0.146722, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16374.470631334445, "interactivity_tok_s": 57.97524573150157, "output_per_gpu": 886.3565998994333, "total_per_gpu": 7989.421598984009, "tpot_error_pct": 35.46, "tpot_ms": 17.248741033910573, "tpot_relative": 1.002921, "ttft_error_pct": 80.43, "ttft_ms": 464.7571752390633, "ttft_relative": 0.137743, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 27127.668195517617, "interactivity_tok_s": 37.419354710083965, "output_per_gpu": 538.1731451610918, "total_per_gpu": 4850.973243084138, "tpot_ms": 26.724138022896334, "tpot_relative": 1.553864, "ttft_ms": 2375.119664151862, "ttft_relative": 0.70393, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.65, "tpot_shape_error_pct": 3.69, "ttft_mape_pct": 64.73, "ttft_shape_error_pct": 6.05}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.27, "tpot_shape_error_pct": 3.6, "ttft_mape_pct": 76.63, "ttft_shape_error_pct": 31.42}, "framework": "trt", "id": "4ec8afc840b21dc6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 2, "moe_tp_size": 1, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 6618.389053644813, "interactivity_tok_s": 163.37167798259378, "output_per_gpu": 302.1532714219535, "total_per_gpu": null, "tpot_error_pct": 29.52, "tpot_ms": 6.121011991482047, "tpot_relative": 0.704794, "ttft_error_pct": 65.25, "ttft_ms": 356.59378635867995, "ttft_relative": 0.347477, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6051.151003299965, "interactivity_tok_s": 156.85030540920567, "output_per_gpu": 297.79232101433814, "total_per_gpu": 2676.975366640371, "tpot_error_pct": 26.59, "tpot_ms": 6.375505596824354, "tpot_relative": 0.734097, "ttft_error_pct": 81.32, "ttft_ms": 191.6509633249998, "ttft_relative": 0.186751, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 9019.74988610018, "interactivity_tok_s": 115.14337917591607, "output_per_gpu": 200.80119797127114, "total_per_gpu": 1805.0830146660755, "tpot_ms": 8.68482414844018, "tpot_relative": 1.0, "ttft_ms": 1026.2379905674607, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9245.58620838748, "interactivity_tok_s": 115.55337622336458, "output_per_gpu": 437.22045017483106, "total_per_gpu": null, "tpot_error_pct": 28.94, "tpot_ms": 8.654009365048761, "tpot_relative": 0.996452, "ttft_error_pct": 64.42, "ttft_ms": 392.53462794259747, "ttft_relative": 0.382499, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8416.551790762485, "interactivity_tok_s": 113.07459565214802, "output_per_gpu": 434.164281876243, "total_per_gpu": 3862.0920184430092, "tpot_error_pct": 27.38, "tpot_ms": 8.84371944230785, "tpot_relative": 1.018296, "ttft_error_pct": 79.58, "ttft_ms": 225.25350279999967, "ttft_relative": 0.219494, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 12429.916388948914, "interactivity_tok_s": 82.11268606133419, "output_per_gpu": 294.9014949743007, "total_per_gpu": 2623.285142308892, "tpot_ms": 12.17838616621369, "tpot_relative": 1.402261, "ttft_ms": 1103.126850060653, "ttft_relative": 1.074923, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12229.680264081255, "interactivity_tok_s": 86.94818991949225, "output_per_gpu": 667.1031383127241, "total_per_gpu": null, "tpot_error_pct": 30.04, "tpot_ms": 11.50110198873522, "tpot_relative": 1.324276, "ttft_error_pct": 63.46, "ttft_ms": 464.05292960512537, "ttft_relative": 0.452188, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11012.297549606234, "interactivity_tok_s": 85.36134254964037, "output_per_gpu": 649.4572594217157, "total_per_gpu": 5861.189209026414, "tpot_error_pct": 28.74, "tpot_ms": 11.714904781616665, "tpot_relative": 1.348894, "ttft_error_pct": 77.05, "ttft_ms": 291.49488274374954, "ttft_relative": 0.284042, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 16351.96093998966, "interactivity_tok_s": 60.83102824886627, "output_per_gpu": 440.0629504555486, "total_per_gpu": 3971.4579813905207, "tpot_ms": 16.438979066881668, "tpot_relative": 1.89284, "ttft_ms": 1269.9310178228188, "ttft_relative": 1.237462, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17054.413893534755, "interactivity_tok_s": 62.197455077117134, "output_per_gpu": 966.0563526078071, "total_per_gpu": null, "tpot_error_pct": 28.86, "tpot_ms": 16.077828244903653, "tpot_relative": 1.851255, "ttft_error_pct": 62.37, "ttft_ms": 606.7955989983166, "ttft_relative": 0.591282, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15447.788892631168, "interactivity_tok_s": 61.5619781834189, "output_per_gpu": 939.8819864862422, "total_per_gpu": 8404.723988150316, "tpot_error_pct": 28.13, "tpot_ms": 16.24379250810592, "tpot_relative": 1.870365, "ttft_error_pct": 74.71, "ttft_ms": 407.82570555937457, "ttft_relative": 0.397399, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 22596.58118179068, "interactivity_tok_s": 44.24531264760666, "output_per_gpu": 645.9619553715767, "total_per_gpu": 5776.397483731799, "tpot_ms": 22.60126418281943, "tpot_relative": 2.602386, "ttft_ms": 1612.6139393571066, "ttft_relative": 1.571384, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 26425.582708197973, "interactivity_tok_s": 39.78660900350053, "output_per_gpu": 1249.6105931994086, "total_per_gpu": null, "tpot_error_pct": 20.88, "tpot_ms": 25.134084684422774, "tpot_relative": 2.894023, "ttft_error_pct": 68.15, "ttft_ms": 713.4140760334742, "ttft_relative": 0.695174, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23955.76704882661, "interactivity_tok_s": 39.59398190319548, "output_per_gpu": 1213.8932912260027, "total_per_gpu": 10941.764613568837, "tpot_error_pct": 20.49, "tpot_ms": 25.256363516176027, "tpot_relative": 2.908103, "ttft_error_pct": 70.49, "ttft_ms": 660.9002942406248, "ttft_relative": 0.644003, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "1.3.0rc20", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26016892349", "measured": {"e2e_ms": 31640.732096908323, "interactivity_tok_s": 31.479349362491295, "output_per_gpu": 922.5197852778529, "total_per_gpu": 8315.388522887086, "tpot_ms": 31.76685732874561, "tpot_relative": 3.657743, "ttft_ms": 2239.5733038705657, "ttft_relative": 2.182314, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.7, "tpot_shape_error_pct": 13.02, "ttft_mape_pct": 60.92, "ttft_shape_error_pct": 52.04}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.65, "tpot_shape_error_pct": 12.47, "ttft_mape_pct": 58.36, "ttft_shape_error_pct": 25.64}, "framework": "vllm", "id": "18fcd36ca74dbdec", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": 7665.358043774049, "interactivity_tok_s": 138.51011180627327, "output_per_gpu": 533.8302498894492, "total_per_gpu": null, "tpot_error_pct": 0.94, "tpot_ms": 7.219689501071567, "tpot_relative": 1.009418, "ttft_error_pct": 33.04, "ttft_ms": 279.61568417783553, "ttft_relative": 0.669564, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6595.705119425011, "interactivity_tok_s": 143.9515014884897, "output_per_gpu": 547.5252003719245, "total_per_gpu": 4921.925014782043, "tpot_error_pct": 2.87, "tpot_ms": 6.946784088111507, "tpot_relative": 0.971262, "ttft_error_pct": 49.59, "ttft_ms": 210.52413042499978, "ttft_relative": 0.504118, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 6990.065329952631, "interactivity_tok_s": 139.81460840295654, "output_per_gpu": 516.6757454624576, "total_per_gpu": 4644.606813340073, "tpot_ms": 7.15232843994329, "tpot_relative": 1.0, "ttft_ms": 417.6088360371068, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11210.74487374285, "interactivity_tok_s": 93.83063875662467, "output_per_gpu": 730.0139368230639, "total_per_gpu": null, "tpot_error_pct": 2.27, "tpot_ms": 10.65749965311195, "tpot_relative": 1.490074, "ttft_error_pct": 38.45, "ttft_ms": 308.12272860932495, "ttft_relative": 0.737826, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9816.459034087473, "interactivity_tok_s": 96.91947308459318, "output_per_gpu": 745.4152975047327, "total_per_gpu": 6630.813705994726, "tpot_error_pct": 0.99, "tpot_ms": 10.317843960286297, "tpot_relative": 1.442585, "ttft_error_pct": 48.24, "ttft_ms": 259.097910225, "ttft_relative": 0.620432, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 10155.199391650967, "interactivity_tok_s": 95.9643216323067, "output_per_gpu": 720.0792485221906, "total_per_gpu": 6405.4378364471495, "tpot_ms": 10.420539456648925, "tpot_relative": 1.456944, "ttft_ms": 500.5743789253756, "ttft_relative": 1.198668, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16048.33400897486, "interactivity_tok_s": 65.11311755458543, "output_per_gpu": 1019.9189517644866, "total_per_gpu": null, "tpot_error_pct": 26.12, "tpot_ms": 15.35788851089311, "tpot_relative": 2.147257, "ttft_error_pct": 96.93, "ttft_ms": 337.21406233120825, "ttft_relative": 0.807488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14055.6552127875, "interactivity_tok_s": 66.6920686365199, "output_per_gpu": 1021.3284852799011, "total_per_gpu": 9217.233944115167, "tpot_error_pct": 23.13, "tpot_ms": 14.994286733706293, "tpot_relative": 2.09642, "ttft_error_pct": 96.94, "ttft_ms": 336.20930182499944, "ttft_relative": 0.805082, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 22134.735016984632, "interactivity_tok_s": 82.11787814931054, "output_per_gpu": 624.1696099507129, "total_per_gpu": 5632.9745019753955, "tpot_ms": 12.177616160292812, "tpot_relative": 1.702609, "ttft_ms": 10983.345726845437, "ttft_relative": 26.300559, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22969.716396839736, "interactivity_tok_s": 45.261797390746025, "output_per_gpu": 1425.1808526684267, "total_per_gpu": null, "tpot_error_pct": 11.29, "tpot_ms": 22.093687340053677, "tpot_relative": 3.08902, "ttft_error_pct": 62.56, "ttft_ms": 367.87424796482327, "ttft_relative": 0.880906, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20542.39908186878, "interactivity_tok_s": 46.16014743729636, "output_per_gpu": 1416.5985661021575, "total_per_gpu": 12667.675432965034, "tpot_error_pct": 13.01, "tpot_ms": 21.66370896796622, "tpot_relative": 3.028903, "ttft_error_pct": 50.03, "ttft_ms": 490.9685426187495, "ttft_relative": 1.175666, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 24052.172408386832, "interactivity_tok_s": 40.15283085743889, "output_per_gpu": 1210.6870187291643, "total_per_gpu": 10826.348812680584, "tpot_ms": 24.904844282348662, "tpot_relative": 3.482061, "ttft_ms": 982.6216945657507, "ttft_relative": 2.352972, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37698.73444173789, "interactivity_tok_s": 27.42886126103732, "output_per_gpu": 1736.7161250780114, "total_per_gpu": null, "tpot_error_pct": 12.9, "tpot_ms": 36.45794808917202, "tpot_relative": 5.097354, "ttft_error_pct": 73.6, "ttft_ms": 402.2535465149149, "ttft_relative": 0.96323, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 34274.94647315303, "interactivity_tok_s": 27.547184357592435, "output_per_gpu": 1698.3679696417735, "total_per_gpu": 15308.711799763378, "tpot_error_pct": 13.27, "tpot_ms": 36.30135069410041, "tpot_relative": 5.075459, "ttft_error_pct": 47.02, "ttft_ms": 807.3720279500009, "ttft_relative": 1.933321, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 40132.12944071565, "interactivity_tok_s": 23.891738915310235, "output_per_gpu": 1452.3361354687083, "total_per_gpu": 13091.035471519235, "tpot_ms": 41.85547161488454, "tpot_relative": 5.852006, "ttft_ms": 1523.975830533891, "ttft_relative": 3.64929, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 1}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.27, "tpot_shape_error_pct": 5.63, "ttft_mape_pct": 46.29, "ttft_shape_error_pct": 34.05}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.33, "tpot_shape_error_pct": 5.15, "ttft_mape_pct": 56.07, "ttft_shape_error_pct": 5.5}, "framework": "vllm", "id": "87e376a00384d51f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 3763.7804935277386, "interactivity_tok_s": 279.6768669649143, "output_per_gpu": 135.90059273636817, "total_per_gpu": null, "tpot_error_pct": 8.19, "tpot_ms": 3.575554928271743, "tpot_relative": 0.918069, "ttft_error_pct": 26.19, "ttft_ms": 105.9878019057456, "ttft_relative": 0.738076, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3290.531119725012, "interactivity_tok_s": 284.6758420796612, "output_per_gpu": 136.40039386878217, "total_per_gpu": 1226.158193545875, "tpot_error_pct": 9.81, "tpot_ms": 3.5127673380875386, "tpot_relative": 0.901948, "ttft_error_pct": 57.0, "ttft_ms": 61.75291487500003, "ttft_relative": 0.430034, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 3722.854489623569, "interactivity_tok_s": 256.7626796126937, "output_per_gpu": 120.66102954125886, "total_per_gpu": 1084.670694983649, "tpot_ms": 3.8946470005236793, "tpot_relative": 1.0, "ttft_ms": 143.6001636320725, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 4252.321949599729, "interactivity_tok_s": 247.36026199843675, "output_per_gpu": 240.57444664938762, "total_per_gpu": null, "tpot_error_pct": 9.75, "tpot_ms": 4.042686533079108, "tpot_relative": 1.038011, "ttft_error_pct": 33.25, "ttft_ms": 116.65362625980185, "ttft_relative": 0.81235, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3753.7244352374996, "interactivity_tok_s": 251.6957002657606, "output_per_gpu": 242.36319821988664, "total_per_gpu": 2155.932702165852, "tpot_error_pct": 11.31, "tpot_ms": 3.9730515815094156, "tpot_relative": 1.020131, "ttft_error_pct": 56.74, "ttft_ms": 75.60689183749962, "ttft_relative": 0.52651, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 4322.423413570505, "interactivity_tok_s": 223.2408959569787, "output_per_gpu": 210.70334952100617, "total_per_gpu": 1874.3037104011328, "tpot_ms": 4.479465985447005, "tpot_relative": 1.15016, "ttft_ms": 174.76841871393844, "ttft_relative": 1.217049, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5469.470581058729, "interactivity_tok_s": 191.49973005952455, "output_per_gpu": 374.0764247065306, "total_per_gpu": null, "tpot_error_pct": 8.06, "tpot_ms": 5.221939475784986, "tpot_relative": 1.340799, "ttft_error_pct": 44.57, "ttft_ms": 127.42649733068797, "ttft_relative": 0.88737, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4809.4848123625, "interactivity_tok_s": 194.15318312114218, "output_per_gpu": 369.3088610236265, "total_per_gpu": 3332.920033809292, "tpot_error_pct": 9.32, "tpot_ms": 5.150572264252029, "tpot_relative": 1.322475, "ttft_error_pct": 57.56, "ttft_ms": 97.55384784374994, "ttft_relative": 0.679344, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5427.616656804457, "interactivity_tok_s": 176.0671849836152, "output_per_gpu": 327.1258883419744, "total_per_gpu": 2952.229263632202, "tpot_ms": 5.679650072743879, "tpot_relative": 1.458322, "ttft_ms": 229.87490235245787, "ttft_relative": 1.600798, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6944.771548909708, "interactivity_tok_s": 150.30290248152428, "output_per_gpu": 589.2202459334205, "total_per_gpu": null, "tpot_error_pct": 13.03, "tpot_ms": 6.653231464528259, "tpot_relative": 1.708302, "ttft_error_pct": 59.79, "ttft_ms": 138.51576069729916, "ttft_relative": 0.964593, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6230.292222927835, "interactivity_tok_s": 152.01679837413803, "output_per_gpu": 579.1969634858701, "total_per_gpu": 5179.363667849987, "tpot_error_pct": 14.01, "tpot_ms": 6.578220372322522, "tpot_relative": 1.689041, "ttft_error_pct": 58.85, "ttft_ms": 141.73635826250046, "ttft_relative": 0.987021, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 7427.904372426565, "interactivity_tok_s": 130.7130509140337, "output_per_gpu": 486.218414764048, "total_per_gpu": 4347.91988016001, "tpot_ms": 7.650345493486125, "tpot_relative": 1.964323, "ttft_ms": 344.47015679324977, "ttft_relative": 2.398815, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9799.449107453387, "interactivity_tok_s": 106.01663517485349, "output_per_gpu": 835.1489874849507, "total_per_gpu": null, "tpot_error_pct": 22.33, "tpot_ms": 9.432481971821664, "tpot_relative": 2.421909, "ttft_error_pct": 67.67, "ttft_ms": 150.02005027982597, "ttft_relative": 1.044707, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8945.096461540623, "interactivity_tok_s": 105.81839649516449, "output_per_gpu": 809.4346549172606, "total_per_gpu": 7296.064265438843, "tpot_error_pct": 22.19, "tpot_ms": 9.450152649456339, "tpot_relative": 2.426447, "ttft_error_pct": 50.22, "ttft_ms": 231.02787162812479, "ttft_relative": 1.608827, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "19019430552", "measured": {"e2e_ms": 11667.864969391667, "interactivity_tok_s": 82.33930820937442, "output_per_gpu": 621.4664519352662, "total_per_gpu": 5601.76061722666, "tpot_ms": 12.144867642768814, "tpot_relative": 3.118349, "ttft_ms": 464.07042949431343, "ttft_relative": 3.231685, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.93, "tpot_shape_error_pct": 6.0, "ttft_mape_pct": 47.68, "ttft_shape_error_pct": 37.89}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.32, "tpot_shape_error_pct": 5.08, "ttft_mape_pct": 53.13, "ttft_shape_error_pct": 4.04}, "framework": "vllm", "id": "05ad116f7b115403", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 4211.537702088153, "interactivity_tok_s": 250.88622999010744, "output_per_gpu": 242.90415339099988, "total_per_gpu": null, "tpot_error_pct": 8.65, "tpot_ms": 3.9858704084294723, "tpot_relative": 0.913519, "ttft_error_pct": 24.92, "ttft_ms": 133.99227426480317, "ttft_relative": 0.750774, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 3667.51488497501, "interactivity_tok_s": 256.66136343442713, "output_per_gpu": 245.02095547989492, "total_per_gpu": 2202.5922626083584, "tpot_error_pct": 10.7, "tpot_ms": 3.896184398846942, "tpot_relative": 0.892964, "ttft_error_pct": 51.64, "ttft_ms": 86.30120285000021, "ttft_relative": 0.483555, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 4188.002538122237, "interactivity_tok_s": 229.18934059637175, "output_per_gpu": 214.70785663493993, "total_per_gpu": 1930.0955823109102, "tpot_ms": 4.36320466474535, "tpot_relative": 1.0, "ttft_ms": 178.47228958271444, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5318.337448161336, "interactivity_tok_s": 197.8415854206526, "output_per_gpu": 384.70669075489866, "total_per_gpu": null, "tpot_error_pct": 3.39, "tpot_ms": 5.054549061936553, "tpot_relative": 1.158449, "ttft_error_pct": 37.99, "ttft_ms": 147.53375780024246, "ttft_relative": 0.826648, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4675.508691875001, "interactivity_tok_s": 202.633446672053, "output_per_gpu": 389.9495093221596, "total_per_gpu": 3468.7811743531825, "tpot_error_pct": 5.67, "tpot_ms": 4.935019447299956, "tpot_relative": 1.131054, "ttft_error_pct": 55.53, "ttft_ms": 105.79543770000012, "ttft_relative": 0.592784, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5083.958245185204, "interactivity_tok_s": 191.14355127463216, "output_per_gpu": 358.6828612729634, "total_per_gpu": 3190.649884159467, "tpot_ms": 5.23167008947749, "tpot_relative": 1.199043, "ttft_ms": 237.91034045862034, "ttft_relative": 1.333038, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6738.276408865696, "interactivity_tok_s": 155.54271540569334, "output_per_gpu": 607.2769580387155, "total_per_gpu": null, "tpot_error_pct": 9.22, "tpot_ms": 6.429102111222348, "tpot_relative": 1.473482, "ttft_error_pct": 46.74, "ttft_ms": 161.30494908523355, "ttft_relative": 0.903809, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5917.660400793657, "interactivity_tok_s": 158.2574080567329, "output_per_gpu": 601.8935108316679, "total_per_gpu": 5431.938283068541, "tpot_error_pct": 10.78, "tpot_ms": 6.31881952496982, "tpot_relative": 1.448206, "ttft_error_pct": 54.99, "ttft_ms": 136.33148636249996, "ttft_relative": 0.76388, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 6785.367102728924, "interactivity_tok_s": 141.1951061554083, "output_per_gpu": 525.4573708582608, "total_per_gpu": 4742.121251551061, "tpot_ms": 7.082398443040487, "tpot_relative": 1.62321, "ttft_ms": 302.8883691004012, "ttft_relative": 1.697117, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8954.73802436644, "interactivity_tok_s": 116.52400224561156, "output_per_gpu": 913.929584286083, "total_per_gpu": null, "tpot_error_pct": 16.15, "tpot_ms": 8.581922871926254, "tpot_relative": 1.966885, "ttft_error_pct": 56.98, "ttft_ms": 175.43092638588095, "ttft_relative": 0.982959, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8033.513802131255, "interactivity_tok_s": 118.13851046327163, "output_per_gpu": 900.9212737717535, "total_per_gpu": 8056.324889694257, "tpot_error_pct": 17.29, "tpot_ms": 8.464640328361787, "tpot_relative": 1.940005, "ttft_error_pct": 51.44, "ttft_ms": 198.03266449374988, "ttft_relative": 1.109599, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 9886.268218085752, "interactivity_tok_s": 97.70669509788516, "output_per_gpu": 732.9003752424743, "total_per_gpu": 6553.828516017588, "tpot_ms": 10.234713179052607, "tpot_relative": 2.345687, "ttft_ms": 407.8344904090045, "ttft_relative": 2.285142, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13868.935754515443, "interactivity_tok_s": 74.78825107859525, "output_per_gpu": 1180.1914934007038, "total_per_gpu": null, "tpot_error_pct": 17.22, "tpot_ms": 13.37108416867639, "tpot_relative": 3.06451, "ttft_error_pct": 71.77, "ttft_ms": 190.3166499594974, "ttft_relative": 1.066365, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12666.445409028098, "interactivity_tok_s": 74.70494187282992, "output_per_gpu": 1146.3180986600285, "total_per_gpu": 10332.65683109917, "tpot_error_pct": 17.13, "tpot_ms": 13.385995289338396, "tpot_relative": 3.067927, "ttft_error_pct": 52.03, "ttft_ms": 323.4629689468741, "ttft_relative": 1.812399, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 15575.09619698394, "interactivity_tok_s": 61.90613028206255, "output_per_gpu": 933.3392503199931, "total_per_gpu": 8412.912778595155, "tpot_ms": 16.15348908813563, "tpot_relative": 3.702208, "ttft_ms": 674.243960592139, "ttft_relative": 3.777864, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.21, "tpot_shape_error_pct": 8.98, "ttft_mape_pct": 69.13, "ttft_shape_error_pct": 70.27}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.23, "tpot_shape_error_pct": 8.55, "ttft_mape_pct": 69.4, "ttft_shape_error_pct": 48.36}, "framework": "vllm", "id": "b8e251adec606289", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 2, "pp_size": 1, "tp_size": 2}, "points": [{"aic": {"e2e_ms": 5678.573312999683, "interactivity_tok_s": 186.36632656022886, "output_per_gpu": 360.30176722667136, "total_per_gpu": null, "tpot_error_pct": 0.1, "tpot_ms": 5.365776202477358, "tpot_relative": 1.001036, "ttft_error_pct": 29.5, "ttft_ms": 189.38425786534674, "ttft_relative": 0.704959, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4912.662802624998, "interactivity_tok_s": 192.30483973706276, "output_per_gpu": 366.94223010952066, "total_per_gpu": 3298.591809343444, "tpot_error_pct": 2.99, "tpot_ms": 5.20007713465399, "tpot_relative": 0.970123, "ttft_error_pct": 50.49, "ttft_ms": 132.99796297499998, "ttft_relative": 0.495068, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 5194.930908433162, "interactivity_tok_s": 186.55935780255032, "output_per_gpu": 346.880164530463, "total_per_gpu": 3118.2458045299913, "tpot_ms": 5.360224283460359, "tpot_relative": 1.0, "ttft_ms": 268.6457673320547, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7556.823607723336, "interactivity_tok_s": 139.2176946742575, "output_per_gpu": 541.4973555579403, "total_per_gpu": null, "tpot_error_pct": 10.26, "tpot_ms": 7.182994965832517, "tpot_relative": 1.340055, "ttft_error_pct": 97.84, "ttft_ms": 208.6197576766707, "ttft_relative": 0.776561, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6627.754788637479, "interactivity_tok_s": 143.27087866633696, "output_per_gpu": 551.1308228090581, "total_per_gpu": 4902.56347825388, "tpot_error_pct": 7.14, "tpot_ms": 6.979785489617164, "tpot_relative": 1.302144, "ttft_error_pct": 98.31, "ttft_ms": 163.5210147374999, "ttft_relative": 0.608686, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 15656.430098132909, "interactivity_tok_s": 153.50026267717618, "output_per_gpu": 215.6675872716752, "total_per_gpu": 1919.760221037343, "tpot_ms": 6.5146468322538515, "tpot_relative": 1.215368, "ttft_ms": 9671.777906790589, "ttft_relative": 36.001974, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10399.65268198314, "interactivity_tok_s": 100.57562847486436, "output_per_gpu": 786.9493578548404, "total_per_gpu": null, "tpot_error_pct": 3.37, "tpot_ms": 9.94276660423671, "tpot_relative": 1.854916, "ttft_error_pct": 47.38, "ttft_ms": 228.20244584898705, "ttft_relative": 0.849455, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9117.398943450014, "interactivity_tok_s": 102.737628359765, "output_per_gpu": 784.8705375434495, "total_per_gpu": 7083.260150527171, "tpot_error_pct": 5.4, "tpot_ms": 9.73353206576091, "tpot_relative": 1.815881, "ttft_error_pct": 51.18, "ttft_ms": 211.7149690062498, "ttft_relative": 0.788082, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 9851.537720189663, "interactivity_tok_s": 97.19090121545595, "output_per_gpu": 726.5822025272298, "total_per_gpu": 6557.222516405748, "tpot_ms": 10.289028988250324, "tpot_relative": 1.919515, "ttft_ms": 433.6882998177316, "ttft_relative": 1.61435, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14323.908075143709, "interactivity_tok_s": 72.67941225655137, "output_per_gpu": 1142.7049038665227, "total_per_gpu": null, "tpot_error_pct": 10.93, "tpot_ms": 13.759054578896368, "tpot_relative": 2.56688, "ttft_error_pct": 97.7, "ttft_ms": 248.39524093272456, "ttft_relative": 0.92462, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12824.874858240593, "interactivity_tok_s": 73.95219688513023, "output_per_gpu": 1132.0788258543362, "total_per_gpu": 10123.409322596117, "tpot_error_pct": 9.02, "tpot_ms": 13.522248724446921, "tpot_relative": 2.522702, "ttft_error_pct": 97.15, "ttft_ms": 308.6238435562502, "ttft_relative": 1.148813, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 22296.80078022211, "interactivity_tok_s": 80.62213623583993, "output_per_gpu": 618.9053526075801, "total_per_gpu": 5534.448726804809, "tpot_ms": 12.403541343468618, "tpot_relative": 2.313997, "ttft_ms": 10821.730387904972, "ttft_relative": 40.282527, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22904.277242021584, "interactivity_tok_s": 45.19763254300129, "output_per_gpu": 1429.2526960833559, "total_per_gpu": null, "tpot_error_pct": 11.4, "tpot_ms": 22.125052657317706, "tpot_relative": 4.127636, "ttft_error_pct": 73.23, "ttft_ms": 270.348373585571, "ttft_relative": 1.006338, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20858.201684031235, "interactivity_tok_s": 45.30198602593345, "output_per_gpu": 1394.1157367024018, "total_per_gpu": 12566.249723370278, "tpot_error_pct": 11.6, "tpot_ms": 22.07408742361853, "tpot_relative": 4.118128, "ttft_error_pct": 49.89, "ttft_ms": 506.18938479843763, "ttft_relative": 1.884226, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.24.0", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26696231118", "measured": {"e2e_ms": 24045.82318119792, "interactivity_tok_s": 40.046221914835726, "output_per_gpu": 1210.7420554229893, "total_per_gpu": 10913.360073690708, "tpot_ms": 24.97114464696943, "tpot_relative": 4.658601, "ttft_ms": 1010.0655517409905, "ttft_relative": 3.759842, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp4", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 2}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp4"], "rows": 98}]}, {"aic": {"coverage_pct": 100.0, "points": 30, "status_counts": {"failed": 0, "success": 30, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.02, "tpot_shape_error_pct": 3.29, "ttft_mape_pct": 46.0, "ttft_shape_error_pct": 24.76}, "aisimulate": {"coverage_pct": 100.0, "points": 30, "status_counts": {"failed": 0, "success": 30, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.24, "tpot_shape_error_pct": 3.48, "ttft_mape_pct": 67.96, "ttft_shape_error_pct": 16.04}, "frameworks": ["sglang"], "gpu_skus": ["b200", "b300"], "hf_model_paths": ["Qwen/Qwen3.5-397B-A17B"], "model": "Qwen/Qwen3.5-397B-A17B", "precisions": ["bf16"], "rows": 30, "workloads": [{"aic": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.79, "tpot_shape_error_pct": 4.08, "ttft_mape_pct": 63.1, "ttft_shape_error_pct": 15.97}, "aisimulate": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.76, "tpot_shape_error_pct": 4.42, "ttft_mape_pct": 81.5, "ttft_shape_error_pct": 17.59}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.3, "tpot_shape_error_pct": 5.49, "ttft_mape_pct": 74.03, "ttft_shape_error_pct": 5.57}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.4, "tpot_shape_error_pct": 5.74, "ttft_mape_pct": 86.83, "ttft_shape_error_pct": 7.3}, "gpu": "b200", "precisions": ["bf16"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.3, "tpot_shape_error_pct": 5.49, "ttft_mape_pct": 74.03, "ttft_shape_error_pct": 5.57}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.4, "tpot_shape_error_pct": 5.74, "ttft_mape_pct": 86.83, "ttft_shape_error_pct": 7.3}, "framework": "sglang", "id": "c34d6c974bf30716", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5917.2617156289325, "interactivity_tok_s": 174.95838076487465, "output_per_gpu": 86.13515159013416, "total_per_gpu": null, "tpot_error_pct": 3.71, "tpot_ms": 5.715645032997265, "tpot_relative": 0.962871, "ttft_error_pct": 73.19, "ttft_ms": 70.15684687273136, "ttft_relative": 0.268125, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5301.265475724994, "interactivity_tok_s": 174.12257320091226, "output_per_gpu": 84.04098577798281, "total_per_gpu": 168.903860299715, "tpot_error_pct": 3.25, "tpot_ms": 5.7430807598170786, "tpot_relative": 0.967493, "ttft_error_pct": 86.21, "ttft_ms": 36.07123355000005, "ttft_relative": 0.137857, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 5705.317651751466, "interactivity_tok_s": 168.462332580696, "output_per_gpu": 78.19144207641956, "total_per_gpu": 157.14756659326068, "tpot_ms": 5.936045077144975, "tpot_relative": 1.0, "ttft_ms": 261.65758902534435, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7538.843487754415, "interactivity_tok_s": 137.10028541148313, "output_per_gpu": 135.49717260616652, "total_per_gpu": null, "tpot_error_pct": 7.23, "tpot_ms": 7.293930840469591, "tpot_relative": 1.228753, "ttft_error_pct": 74.15, "ttft_ms": 77.15223795402267, "ttft_relative": 0.29486, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6846.223912949987, "interactivity_tok_s": 136.40652874567397, "output_per_gpu": 133.24846075511888, "total_per_gpu": 265.5126978878908, "tpot_error_pct": 6.76, "tpot_ms": 7.331027401661039, "tpot_relative": 1.235002, "ttft_error_pct": 86.67, "ttft_ms": 39.77822013749967, "ttft_relative": 0.152024, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 7604.594523412652, "interactivity_tok_s": 127.18722747662018, "output_per_gpu": 120.00993957615364, "total_per_gpu": 239.13344026379843, "tpot_ms": 7.862424709146381, "tpot_relative": 1.324522, "ttft_ms": 298.44391730021016, "ttft_relative": 1.14059, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9199.208656439518, "interactivity_tok_s": 112.3180278593366, "output_per_gpu": 222.54999705962246, "total_per_gpu": null, "tpot_error_pct": 10.79, "tpot_ms": 8.90329022917289, "tpot_relative": 1.499869, "ttft_error_pct": 76.7, "ttft_ms": 91.14275199564965, "ttft_relative": 0.348328, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8283.534041612478, "interactivity_tok_s": 111.08792051797033, "output_per_gpu": 216.7208441372214, "total_per_gpu": 435.74875525719915, "tpot_error_pct": 9.8, "tpot_ms": 9.001878830185081, "tpot_relative": 1.516478, "ttft_error_pct": 88.45, "ttft_ms": 45.186572937499896, "ttft_relative": 0.172694, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 9533.630958762842, "interactivity_tok_s": 100.1992241427127, "output_per_gpu": 188.68193002122234, "total_per_gpu": 379.3724432626129, "tpot_ms": 9.98011719707241, "tpot_relative": 1.681274, "ttft_ms": 391.1163377437333, "ttft_relative": 1.494764, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11661.414404891617, "interactivity_tok_s": 88.63304324137836, "output_per_gpu": 352.0523956690958, "total_per_gpu": null, "tpot_error_pct": 12.67, "tpot_ms": 11.282473933301093, "tpot_relative": 1.900672, "ttft_error_pct": 72.32, "ttft_ms": 119.44357112459784, "ttft_relative": 0.456488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10675.214388924931, "interactivity_tok_s": 86.91124259651237, "output_per_gpu": 338.6647540869255, "total_per_gpu": 676.2582229928979, "tpot_error_pct": 10.94, "tpot_ms": 11.505991286334787, "tpot_relative": 1.938326, "ttft_error_pct": 87.09, "ttft_ms": 55.7138254781248, "ttft_relative": 0.212926, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 12361.471030462599, "interactivity_tok_s": 77.40285190254578, "output_per_gpu": 293.19937567717557, "total_per_gpu": 585.4712850548997, "tpot_ms": 12.919420608158626, "tpot_relative": 2.176436, "ttft_ms": 431.4421721908275, "ttft_relative": 1.648881, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16858.13530088961, "interactivity_tok_s": 61.19424790350063, "output_per_gpu": 487.4262304641093, "total_per_gpu": null, "tpot_error_pct": 2.11, "tpot_ms": 16.341405185286945, "tpot_relative": 2.752911, "ttft_error_pct": 73.82, "ttft_ms": 140.87779634106687, "ttft_relative": 0.538405, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15644.773757157815, "interactivity_tok_s": 59.178046966654264, "output_per_gpu": 461.51800752599434, "total_per_gpu": 923.2441030272987, "tpot_error_pct": 1.23, "tpot_ms": 16.89815820659782, "tpot_relative": 2.846703, "ttft_error_pct": 85.75, "ttft_ms": 76.69910819843744, "ttft_relative": 0.293128, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 15919.54864321392, "interactivity_tok_s": 59.90383258262537, "output_per_gpu": 452.35450802812545, "total_per_gpu": 904.9129724180028, "tpot_ms": 16.693422722506106, "tpot_relative": 2.812213, "ttft_ms": 538.1686374451192, "ttft_relative": 2.056767, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.03, "tpot_shape_error_pct": 3.37, "ttft_mape_pct": 57.63, "ttft_shape_error_pct": 21.17}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.44, "tpot_shape_error_pct": 3.77, "ttft_mape_pct": 78.83, "ttft_shape_error_pct": 22.73}, "gpu": "b300", "precisions": ["bf16"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.36, "tpot_shape_error_pct": 3.81, "ttft_mape_pct": 52.98, "ttft_shape_error_pct": 29.23}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.24, "tpot_shape_error_pct": 4.3, "ttft_mape_pct": 76.69, "ttft_shape_error_pct": 31.83}, "framework": "sglang", "id": "efae091b7b68dc90", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7518.769279018642, "interactivity_tok_s": 137.80589209427583, "output_per_gpu": 135.52519122314277, "total_per_gpu": null, "tpot_error_pct": 3.88, "tpot_ms": 7.256583770132844, "tpot_relative": 1.038818, "ttft_error_pct": 41.38, "ttft_ms": 95.28408217274304, "ttft_relative": 0.586192, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6734.204125600018, "interactivity_tok_s": 137.13247244958868, "output_per_gpu": 132.7077720265594, "total_per_gpu": 266.7133753794294, "tpot_error_pct": 4.39, "tpot_ms": 7.292218846033061, "tpot_relative": 1.043919, "ttft_error_pct": 70.05, "ttft_ms": 48.6804265249998, "ttft_relative": 0.299484, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 6568.4004159178585, "interactivity_tok_s": 143.15520587601173, "output_per_gpu": 135.8618403297597, "total_per_gpu": 273.052357569225, "tpot_ms": 6.985425321284584, "tpot_relative": 1.0, "ttft_ms": 162.54768197541125, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9797.55985994475, "interactivity_tok_s": 105.54229582370853, "output_per_gpu": 208.48764371843856, "total_per_gpu": null, "tpot_error_pct": 6.67, "tpot_ms": 9.474874430155845, "tpot_relative": 1.356378, "ttft_error_pct": 36.11, "ttft_ms": 104.76331789531993, "ttft_relative": 0.644508, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8903.324372750032, "interactivity_tok_s": 104.91040392455369, "output_per_gpu": 205.1337916970865, "total_per_gpu": 408.75238747831986, "tpot_error_pct": 7.31, "tpot_ms": 9.531943092309033, "tpot_relative": 1.364547, "ttft_error_pct": 67.59, "ttft_ms": 53.14861796249998, "ttft_relative": 0.326972, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 8411.346193853387, "interactivity_tok_s": 112.58112028135095, "output_per_gpu": 216.71092621849317, "total_per_gpu": 431.82114341868925, "tpot_ms": 8.882484003542555, "tpot_relative": 1.271574, "ttft_ms": 163.97242512612138, "ttft_relative": 1.008765, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13057.95360056734, "interactivity_tok_s": 79.09275178878262, "output_per_gpu": 313.58690913460646, "total_per_gpu": null, "tpot_error_pct": 3.55, "tpot_ms": 12.643383589314256, "tpot_relative": 1.809966, "ttft_error_pct": 59.39, "ttft_ms": 123.77218869885687, "ttft_relative": 0.761452, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11747.234959468778, "interactivity_tok_s": 78.30750945163997, "output_per_gpu": 306.176188106502, "total_per_gpu": 615.611725710736, "tpot_error_pct": 4.59, "tpot_ms": 12.770167344136588, "tpot_relative": 1.828116, "ttft_error_pct": 80.4, "ttft_ms": 59.750316775000194, "ttft_relative": 0.367586, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 11480.744316620985, "interactivity_tok_s": 81.89947268208415, "output_per_gpu": 313.30136898875065, "total_per_gpu": 629.9379374453874, "tpot_ms": 12.210090825392507, "tpot_relative": 1.747938, "ttft_ms": 304.7986523626605, "ttft_relative": 1.875134, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 17146.339193456086, "interactivity_tok_s": 60.23270813577854, "output_per_gpu": 478.82115614303194, "total_per_gpu": null, "tpot_error_pct": 1.98, "tpot_ms": 16.602275257917466, "tpot_relative": 2.376702, "ttft_error_pct": 62.87, "ttft_ms": 162.21160460652072, "ttft_relative": 0.997932, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15709.619868865737, "interactivity_tok_s": 59.02142412998217, "output_per_gpu": 461.2520765406054, "total_per_gpu": 921.0450921416867, "tpot_error_pct": 0.03, "tpot_ms": 16.943000185792062, "tpot_relative": 2.425479, "ttft_error_pct": 83.37, "ttft_ms": 72.6381444656252, "ttft_relative": 0.446873, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 16070.200011904308, "interactivity_tok_s": 59.04106386945309, "output_per_gpu": 450.84274551895925, "total_per_gpu": 900.2593575345464, "tpot_ms": 16.93736417438413, "tpot_relative": 2.424672, "ttft_ms": 436.85279777510004, "ttft_relative": 2.687536, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25753.845705675096, "interactivity_tok_s": 40.01909479632582, "output_per_gpu": 637.9767491291888, "total_per_gpu": null, "tpot_error_pct": 10.74, "tpot_ms": 24.98807144662879, "tpot_relative": 3.577173, "ttft_error_pct": 65.14, "ttft_ms": 191.04861577384244, "ttft_relative": 1.175339, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23981.972109784707, "interactivity_tok_s": 38.57119180596316, "output_per_gpu": 603.8044944118747, "total_per_gpu": 1207.8812305405975, "tpot_error_pct": 14.89, "tpot_ms": 25.926085069671053, "tpot_relative": 3.711454, "ttft_error_pct": 82.05, "ttft_ms": 98.38613206875027, "ttft_relative": 0.605276, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 21331.86839217742, "interactivity_tok_s": 44.31597457076329, "output_per_gpu": 675.5437646691353, "total_per_gpu": 1351.3921166608256, "tpot_ms": 22.565226415210848, "tpot_relative": 3.23033, "ttft_ms": 548.0291704319825, "ttft_relative": 3.371498, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.7, "tpot_shape_error_pct": 2.93, "ttft_mape_pct": 62.28, "ttft_shape_error_pct": 13.12}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.64, "tpot_shape_error_pct": 3.23, "ttft_mape_pct": 80.97, "ttft_shape_error_pct": 13.64}, "framework": "sglang", "id": "d0ba960699a7a974", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5765.95591167407, "interactivity_tok_s": 179.59745500394803, "output_per_gpu": 88.38463146585576, "total_per_gpu": null, "tpot_error_pct": 4.42, "tpot_ms": 5.5680076311661395, "tpot_relative": 0.955839, "ttft_error_pct": 64.85, "ttft_ms": 69.88410499110941, "ttft_relative": 0.351456, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5165.787616375005, "interactivity_tok_s": 178.71131810112328, "output_per_gpu": 86.17385796305821, "total_per_gpu": 173.19046334524356, "tpot_error_pct": 3.94, "tpot_ms": 5.595616498302323, "tpot_relative": 0.960579, "ttft_error_pct": 82.02, "ttft_ms": 35.758155500000086, "ttft_relative": 0.179833, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 5540.103630899102, "interactivity_tok_s": 171.66627561178814, "output_per_gpu": 80.43102591598887, "total_per_gpu": 161.64863654700238, "tpot_ms": 5.825255988319065, "tpot_relative": 1.0, "ttft_ms": 198.84141764923697, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7405.148811700623, "interactivity_tok_s": 139.59532744215258, "output_per_gpu": 137.93699608659597, "total_per_gpu": null, "tpot_error_pct": 3.93, "tpot_ms": 7.163563554191265, "tpot_relative": 1.229742, "ttft_error_pct": 50.32, "ttft_ms": 76.82329576295871, "ttft_relative": 0.386355, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6725.409131037483, "interactivity_tok_s": 138.86343820519974, "output_per_gpu": 135.6155860518086, "total_per_gpu": 270.2294639968653, "tpot_error_pct": 3.42, "tpot_ms": 7.201319605253407, "tpot_relative": 1.236224, "ttft_error_pct": 74.5, "ttft_ms": 39.43794331249999, "ttft_relative": 0.198339, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 7078.494479475194, "interactivity_tok_s": 134.11332284513742, "output_per_gpu": 128.70322017137056, "total_per_gpu": 256.4557895896523, "tpot_ms": 7.456380759089195, "tpot_relative": 1.280009, "ttft_ms": 154.63806286206818, "ttft_relative": 0.777695, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9235.568815458028, "interactivity_tok_s": 111.86716975868814, "output_per_gpu": 221.67610446532558, "total_per_gpu": null, "tpot_error_pct": 5.97, "tpot_ms": 8.939173147556415, "tpot_relative": 1.534555, "ttft_error_pct": 65.75, "ttft_ms": 90.79468550781606, "ttft_relative": 0.456619, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8312.545821331283, "interactivity_tok_s": 110.69382112286664, "output_per_gpu": 216.0327024177282, "total_per_gpu": 434.36514631592047, "tpot_error_pct": 4.97, "tpot_ms": 9.033927909038677, "tpot_relative": 1.550821, "ttft_error_pct": 83.11, "ttft_ms": 44.769055987499826, "ttft_relative": 0.22515, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 8968.258117943697, "interactivity_tok_s": 105.1871810419506, "output_per_gpu": 200.3226459781038, "total_per_gpu": 402.77779455084425, "tpot_ms": 9.506861863720651, "tpot_relative": 1.632008, "ttft_ms": 265.1150878180488, "ttft_relative": 1.333299, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11666.92978015611, "interactivity_tok_s": 88.588113227773, "output_per_gpu": 351.8844323875861, "total_per_gpu": null, "tpot_error_pct": 8.51, "tpot_ms": 11.288196164973666, "tpot_relative": 1.937803, "ttft_error_pct": 63.72, "ttft_ms": 119.10510338804872, "ttft_relative": 0.598995, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10667.96968987498, "interactivity_tok_s": 86.9657904627549, "output_per_gpu": 338.92256122899124, "total_per_gpu": 676.7730217656203, "tpot_error_pct": 6.81, "tpot_ms": 11.498774341943951, "tpot_relative": 1.973952, "ttft_error_pct": 83.2, "ttft_ms": 55.15713267499996, "ttft_relative": 0.277393, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 11717.69451078817, "interactivity_tok_s": 81.04703191300439, "output_per_gpu": 308.75786891340084, "total_per_gpu": 616.539055944563, "tpot_ms": 12.338514766011377, "tpot_relative": 2.118107, "ttft_ms": 328.2613737241263, "ttft_relative": 1.65087, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16776.223946736238, "interactivity_tok_s": 61.49403810144133, "output_per_gpu": 489.80810594005135, "total_per_gpu": null, "tpot_error_pct": 0.66, "tpot_ms": 16.261739038024913, "tpot_relative": 2.791592, "ttft_error_pct": 66.73, "ttft_ms": 140.46491083675116, "ttft_relative": 0.706417, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15561.047718856269, "interactivity_tok_s": 59.494701288410376, "output_per_gpu": 463.96765881172695, "total_per_gpu": 928.144510090827, "tpot_error_pct": 4.04, "tpot_ms": 16.808219527859045, "tpot_relative": 2.885404, "ttft_error_pct": 82.03, "ttft_ms": 75.86393130624992, "ttft_relative": 0.38153, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 15304.666326192819, "interactivity_tok_s": 61.8973828172237, "output_per_gpu": 469.7231046927357, "total_per_gpu": 939.6579968525214, "tpot_ms": 16.155771932278498, "tpot_relative": 2.773401, "ttft_ms": 422.2142068501853, "ttft_relative": 2.123372, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["bf16"], "rows": 15}, {"aic": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.24, "tpot_shape_error_pct": 2.51, "ttft_mape_pct": 28.91, "ttft_shape_error_pct": 33.55}, "aisimulate": {"coverage_pct": 100.0, "points": 15, "status_counts": {"failed": 0, "success": 15, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.73, "tpot_shape_error_pct": 2.53, "ttft_mape_pct": 54.41, "ttft_shape_error_pct": 14.48}, "gpu_skus": ["b200", "b300"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.19, "tpot_shape_error_pct": 1.07, "ttft_mape_pct": 34.92, "ttft_shape_error_pct": 28.97}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.48, "tpot_shape_error_pct": 1.95, "ttft_mape_pct": 60.29, "ttft_shape_error_pct": 7.98}, "gpu": "b200", "precisions": ["bf16"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.19, "tpot_shape_error_pct": 1.07, "ttft_mape_pct": 34.92, "ttft_shape_error_pct": 28.97}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.48, "tpot_shape_error_pct": 1.95, "ttft_mape_pct": 60.29, "ttft_shape_error_pct": 7.98}, "framework": "sglang", "id": "d03c6494f83a08a7", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7235.450818109004, "interactivity_tok_s": 146.86029124673652, "output_per_gpu": 69.62223628612405, "total_per_gpu": null, "tpot_error_pct": 13.75, "tpot_ms": 6.809192542863227, "tpot_relative": 1.137523, "ttft_error_pct": 15.29, "ttft_ms": 269.6468467599236, "ttft_relative": 0.847144, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6477.602899825028, "interactivity_tok_s": 144.92168384258338, "output_per_gpu": 69.37284112652344, "total_per_gpu": 623.6204687112054, "tpot_error_pct": 15.27, "tpot_ms": 6.900278643506644, "tpot_relative": 1.15274, "ttft_error_pct": 57.59, "ttft_ms": 135.00419207499976, "ttft_relative": 0.42414, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 5819.711296474998, "interactivity_tok_s": 167.05697692322317, "output_per_gpu": 77.3074904609687, "total_per_gpu": 694.9482341112295, "tpot_ms": 5.985981659775782, "tpot_relative": 1.0, "ttft_ms": 318.3010235494294, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9462.612155646424, "interactivity_tok_s": 111.60608491597112, "output_per_gpu": 107.19329822288621, "total_per_gpu": null, "tpot_error_pct": 13.59, "tpot_ms": 8.960084934015075, "tpot_relative": 1.496845, "ttft_error_pct": 29.57, "ttft_ms": 296.4452681490027, "ttft_relative": 0.931336, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8404.105305737496, "interactivity_tok_s": 112.31046891912855, "output_per_gpu": 108.51315130092058, "total_per_gpu": 965.2746507020205, "tpot_error_pct": 12.88, "tpot_ms": 8.903889455933715, "tpot_relative": 1.487457, "ttft_error_pct": 62.01, "ttft_ms": 159.90913186250003, "ttft_relative": 0.502383, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 7728.883496424896, "interactivity_tok_s": 126.7779879629392, "output_per_gpu": 117.98389289482326, "total_per_gpu": 1049.5212758745947, "tpot_ms": 7.887804626559686, "tpot_relative": 1.317713, "ttft_ms": 420.91323638760514, "ttft_relative": 1.322375, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12743.787691645919, "interactivity_tok_s": 82.55022742543325, "output_per_gpu": 160.26993763474928, "total_per_gpu": null, "tpot_error_pct": 14.77, "tpot_ms": 12.113837007939068, "tpot_relative": 2.023701, "ttft_error_pct": 36.17, "ttft_ms": 351.3324325242517, "ttft_relative": 1.103774, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11219.613165568819, "interactivity_tok_s": 83.07438409940958, "output_per_gpu": 158.8960541852616, "total_per_gpu": 1433.997117803856, "tpot_error_pct": 14.04, "tpot_ms": 12.037404921393899, "tpot_relative": 2.010932, "ttft_error_pct": 62.36, "ttft_ms": 207.19121419999973, "ttft_relative": 0.650929, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 10215.928362855539, "interactivity_tok_s": 94.7398585911471, "output_per_gpu": 174.44306346030385, "total_per_gpu": 1574.304985140113, "tpot_ms": 10.555219470144367, "tpot_relative": 1.763323, "ttft_ms": 550.4462199181035, "ttft_relative": 1.729326, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16906.035254846305, "interactivity_tok_s": 62.19826166827505, "output_per_gpu": 243.36777207682593, "total_per_gpu": null, "tpot_error_pct": 10.39, "tpot_ms": 16.077619746566995, "tpot_relative": 2.685879, "ttft_error_pct": 40.64, "ttft_ms": 458.6302541082688, "ttft_relative": 1.44087, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15180.306507803101, "interactivity_tok_s": 62.18096415606162, "output_per_gpu": 238.46898754227897, "total_per_gpu": 2132.4656167945923, "tpot_error_pct": 10.42, "tpot_ms": 16.082092221828574, "tpot_relative": 2.686626, "ttft_error_pct": 61.56, "ttft_ms": 296.9939111718751, "ttft_relative": 0.93306, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 14256.174768406117, "interactivity_tok_s": 68.65854247525537, "output_per_gpu": 253.89222550152888, "total_per_gpu": 2270.385121492915, "tpot_ms": 14.564830011653703, "tpot_relative": 2.433156, "ttft_ms": 772.6545190812885, "ttft_relative": 2.427433, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25258.03482671865, "interactivity_tok_s": 41.37660197844784, "output_per_gpu": 326.3649890623846, "total_per_gpu": null, "tpot_error_pct": 13.44, "tpot_ms": 24.168248531401346, "tpot_relative": 4.037475, "ttft_error_pct": 52.93, "ttft_ms": 533.9165790950725, "ttft_relative": 1.677395, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23023.90049336435, "interactivity_tok_s": 40.8912091153985, "output_per_gpu": 315.15231527461054, "total_per_gpu": 2840.7129984821827, "tpot_error_pct": 14.78, "tpot_ms": 24.455134040617732, "tpot_relative": 4.085401, "ttft_error_pct": 57.96, "ttft_ms": 476.8397968578121, "ttft_relative": 1.498078, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719974515", "measured": {"e2e_ms": 20783.640213207833, "interactivity_tok_s": 46.93627535452399, "output_per_gpu": 348.77869862673873, "total_per_gpu": 3143.813752151407, "tpot_ms": 21.30548264528225, "tpot_relative": 3.55923, "ttft_ms": 1134.2155090626875, "ttft_relative": 3.563342, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.77, "tpot_shape_error_pct": 3.23, "ttft_mape_pct": 25.9, "ttft_shape_error_pct": 35.84}, "aisimulate": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.35, "tpot_shape_error_pct": 2.82, "ttft_mape_pct": 51.47, "ttft_shape_error_pct": 17.73}, "gpu": "b300", "precisions": ["bf16"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.48, "tpot_shape_error_pct": 3.72, "ttft_mape_pct": 26.62, "ttft_shape_error_pct": 37.44}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.32, "tpot_shape_error_pct": 3.12, "ttft_mape_pct": 49.77, "ttft_shape_error_pct": 19.98}, "framework": "sglang", "id": "32d4e2a8d88918e9", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8890.794612271113, "interactivity_tok_s": 119.570698736691, "output_per_gpu": 113.29658223472035, "total_per_gpu": null, "tpot_error_pct": 15.2, "tpot_ms": 8.36325295883835, "tpot_relative": 1.152009, "ttft_error_pct": 19.67, "ttft_ms": 335.18683537948016, "ttft_relative": 1.196696, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7981.79875037498, "interactivity_tok_s": 117.62508289547898, "output_per_gpu": 112.77742786738636, "total_per_gpu": 1013.8018176080456, "tpot_error_pct": 17.11, "tpot_ms": 8.50158805744345, "tpot_relative": 1.171065, "ttft_error_pct": 40.21, "ttft_ms": 167.46170024999918, "ttft_relative": 0.597878, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 6950.830461547594, "interactivity_tok_s": 137.74656685289423, "output_per_gpu": 129.38853264891728, "total_per_gpu": 1163.125742957683, "tpot_ms": 7.25970906460373, "tpot_relative": 1.0, "ttft_ms": 280.09356922702864, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11830.765905901088, "interactivity_tok_s": 89.24797890474721, "output_per_gpu": 171.48417259982557, "total_per_gpu": null, "tpot_error_pct": 18.41, "tpot_ms": 11.204735527594213, "tpot_relative": 1.543414, "ttft_error_pct": 6.44, "ttft_ms": 368.32146117220714, "ttft_relative": 1.314994, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10564.951067525117, "interactivity_tok_s": 89.31969576659671, "output_per_gpu": 172.75146097177503, "total_per_gpu": 1536.7041151110611, "tpot_error_pct": 18.31, "tpot_ms": 11.195738984748921, "tpot_relative": 1.542175, "ttft_error_pct": 42.61, "ttft_ms": 198.592888975, "ttft_relative": 0.709023, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 9103.811819228576, "interactivity_tok_s": 105.67613108991765, "output_per_gpu": 200.30682705595478, "total_per_gpu": 1781.821836354931, "tpot_ms": 9.462874820323622, "tpot_relative": 1.303479, "ttft_ms": 346.05187023989856, "ttft_relative": 1.235487, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16850.500727066224, "interactivity_tok_s": 62.322892205342875, "output_per_gpu": 242.47462183341008, "total_per_gpu": null, "tpot_error_pct": 22.16, "tpot_ms": 16.04546844047573, "tpot_relative": 2.210208, "ttft_error_pct": 24.31, "ttft_ms": 435.9865124595532, "ttft_relative": 1.556575, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14904.043373087576, "interactivity_tok_s": 62.46170919703946, "output_per_gpu": 239.84810269515694, "total_per_gpu": 2164.56909354443, "tpot_error_pct": 21.89, "tpot_ms": 16.00980845473562, "tpot_relative": 2.205296, "ttft_error_pct": 55.21, "ttft_ms": 258.0098565312496, "ttft_relative": 0.921156, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 12601.185275216994, "interactivity_tok_s": 76.13637889329878, "output_per_gpu": 283.3002274759685, "total_per_gpu": 2556.713643751372, "tpot_ms": 13.13432572622673, "tpot_relative": 1.809208, "ttft_ms": 576.0338567433791, "ttft_relative": 2.056577, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22997.386593340132, "interactivity_tok_s": 45.612094269573554, "output_per_gpu": 357.7031743433051, "total_per_gpu": null, "tpot_error_pct": 12.33, "tpot_ms": 21.924009761311698, "tpot_relative": 3.019957, "ttft_error_pct": 34.07, "ttft_ms": 569.1246075182662, "ttft_relative": 2.031909, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20745.43906089699, "interactivity_tok_s": 45.41551093609771, "output_per_gpu": 349.5008769100723, "total_per_gpu": 3125.348124850623, "tpot_error_pct": 12.82, "tpot_ms": 22.018908945163222, "tpot_relative": 3.033029, "ttft_error_pct": 57.05, "ttft_ms": 370.71680725937534, "ttft_relative": 1.323546, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 18923.819753069984, "interactivity_tok_s": 51.23689193055073, "output_per_gpu": 382.8797868030465, "total_per_gpu": 3423.832965191687, "tpot_ms": 19.51718697838765, "tpot_relative": 2.688426, "ttft_ms": 863.1632967870246, "ttft_relative": 3.081696, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35407.38242708294, "interactivity_tok_s": 29.443154562556277, "output_per_gpu": 465.3458512072144, "total_per_gpu": null, "tpot_error_pct": 19.28, "tpot_ms": 33.96375201153647, "tpot_relative": 4.67839, "ttft_error_pct": 48.6, "ttft_ms": 662.4641192811291, "ttft_relative": 2.365153, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32472.830051184446, "interactivity_tok_s": 28.917648251747888, "output_per_gpu": 447.2847485005818, "total_per_gpu": 4031.7254149990376, "tpot_error_pct": 21.45, "tpot_ms": 34.580958703637194, "tpot_relative": 4.763408, "ttft_error_pct": 53.75, "ttft_ms": 596.0574090656249, "ttft_relative": 2.128065, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 27536.653121394556, "interactivity_tok_s": 35.12025225458774, "output_per_gpu": 526.4962566539177, "total_per_gpu": 4745.720362630909, "tpot_ms": 28.473599584392808, "tpot_relative": 3.922141, "ttft_ms": 1288.7369485906675, "ttft_relative": 4.601094, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.07, "tpot_shape_error_pct": 2.74, "ttft_mape_pct": 25.19, "ttft_shape_error_pct": 34.25}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.39, "tpot_shape_error_pct": 2.51, "ttft_mape_pct": 53.18, "ttft_shape_error_pct": 15.48}, "framework": "sglang", "id": "74b7b8a8bd6b07d1", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7073.906358335368, "interactivity_tok_s": 150.2495558852399, "output_per_gpu": 71.20476036098826, "total_per_gpu": null, "tpot_error_pct": 9.67, "tpot_ms": 6.655593716122507, "tpot_relative": 1.09665, "ttft_error_pct": 6.77, "ttft_ms": 265.233986742044, "ttft_relative": 1.067658, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6336.24889634999, "interactivity_tok_s": 148.17191006114848, "output_per_gpu": 70.883989053114, "total_per_gpu": 637.2047873432422, "tpot_error_pct": 11.2, "tpot_ms": 6.748917521460808, "tpot_relative": 1.112027, "ttft_error_pct": 46.56, "ttft_ms": 132.75703207500004, "ttft_relative": 0.534393, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 5826.497514417861, "interactivity_tok_s": 164.77123709110938, "output_per_gpu": 77.13759354659628, "total_per_gpu": 693.4209621752237, "tpot_ms": 6.069020404617435, "tpot_relative": 1.0, "ttft_ms": 248.42590034822933, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9248.795120248946, "interactivity_tok_s": 114.207970230953, "output_per_gpu": 109.66471128330998, "total_per_gpu": null, "tpot_error_pct": 10.15, "tpot_ms": 8.755956331049275, "tpot_relative": 1.44273, "ttft_error_pct": 12.56, "ttft_ms": 291.4517935855387, "ttft_relative": 1.173194, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8220.96857598746, "interactivity_tok_s": 114.82275034244542, "output_per_gpu": 110.9028005030419, "total_per_gpu": 986.5316851833188, "tpot_error_pct": 9.57, "tpot_ms": 8.70907548388814, "tpot_relative": 1.435005, "ttft_error_pct": 52.81, "ttft_ms": 157.3040212875, "ttft_relative": 0.633203, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 7692.38397279114, "interactivity_tok_s": 125.8057803394889, "output_per_gpu": 117.84431964550829, "total_per_gpu": 1048.2797072917579, "tpot_ms": 7.948760361419675, "tpot_relative": 1.309727, "ttft_ms": 333.3351041888818, "ttft_relative": 1.341789, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 12719.874195762703, "interactivity_tok_s": 82.67090887395256, "output_per_gpu": 160.57996925343755, "total_per_gpu": null, "tpot_error_pct": 15.53, "tpot_ms": 12.096153454955832, "tpot_relative": 1.993098, "ttft_error_pct": 25.3, "ttft_ms": 345.5092113428856, "ttft_relative": 1.390794, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11195.156703599881, "interactivity_tok_s": 83.23373353925263, "output_per_gpu": 159.26091751859298, "total_per_gpu": 1437.2899180629454, "tpot_error_pct": 14.75, "tpot_ms": 12.014359532825772, "tpot_relative": 1.979621, "ttft_error_pct": 55.93, "ttft_ms": 203.84860565000025, "ttft_relative": 0.820561, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 10047.525072422286, "interactivity_tok_s": 95.51085189970287, "output_per_gpu": 177.36094636255203, "total_per_gpu": 1600.6381479953375, "tpot_ms": 10.470014455007819, "tpot_relative": 1.725157, "ttft_ms": 462.51472886360716, "ttft_relative": 1.861781, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16789.260267519072, "interactivity_tok_s": 62.613865618865205, "output_per_gpu": 245.05211260690965, "total_per_gpu": null, "tpot_error_pct": 10.18, "tpot_ms": 15.97090341118798, "tpot_relative": 2.631546, "ttft_error_pct": 32.12, "ttft_ms": 451.0260778737677, "ttft_relative": 1.815536, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15078.260285087494, "interactivity_tok_s": 62.58951994458381, "output_per_gpu": 240.05821547514773, "total_per_gpu": 2146.67699899158, "tpot_error_pct": 10.22, "tpot_ms": 15.977115671847153, "tpot_relative": 2.632569, "ttft_error_pct": 56.01, "ttft_ms": 292.2845103625003, "ttft_relative": 1.176546, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 14080.601128465969, "interactivity_tok_s": 68.98704398446999, "output_per_gpu": 256.87726428313965, "total_per_gpu": 2297.0782887352793, "tpot_ms": 14.495475414559214, "tpot_relative": 2.388437, "ttft_ms": 664.4520292566085, "ttft_relative": 2.674649, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24993.34906448882, "interactivity_tok_s": 41.809029540550426, "output_per_gpu": 329.8101611024883, "total_per_gpu": null, "tpot_error_pct": 14.8, "tpot_ms": 23.91827820423585, "tpot_relative": 3.941044, "ttft_error_pct": 49.21, "ttft_ms": 524.9504615555436, "ttft_relative": 2.113107, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22792.686793300258, "interactivity_tok_s": 41.300741809793735, "output_per_gpu": 318.3304731966092, "total_per_gpu": 2869.360208363487, "tpot_error_pct": 16.22, "tpot_ms": 24.212640165287972, "tpot_relative": 3.989547, "ttft_error_pct": 54.59, "ttft_ms": 469.41389092499986, "ttft_relative": 1.889553, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26015573402", "measured": {"e2e_ms": 20240.69707126191, "interactivity_tok_s": 47.9987622091923, "output_per_gpu": 357.9290642559056, "total_per_gpu": 3226.2931163311982, "tpot_ms": 20.83387058278117, "tpot_relative": 3.432823, "ttft_ms": 1033.6237568008073, "ttft_relative": 4.160692, "unavailable_metrics": {}}, "status": "success"}], "precision": "bf16", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["bf16"], "rows": 15}]}, {"aic": {"coverage_pct": 84.78, "points": 78, "status_counts": {"failed": 14, "success": 78, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.42, "tpot_shape_error_pct": 6.12, "ttft_mape_pct": 55.45, "ttft_shape_error_pct": 36.55}, "aisimulate": {"coverage_pct": 76.09, "points": 70, "status_counts": {"failed": 22, "success": 70, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.39, "tpot_shape_error_pct": 9.91, "ttft_mape_pct": 62.9, "ttft_shape_error_pct": 26.01}, "frameworks": ["dynamo-sglang", "sglang"], "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "hf_model_paths": ["Qwen/Qwen3.5-397B-A17B-FP8"], "model": "Qwen/Qwen3.5-397B-A17B-FP8", "precisions": ["fp8"], "rows": 92, "workloads": [{"aic": {"coverage_pct": 75.76, "points": 25, "status_counts": {"failed": 8, "success": 25, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.25, "tpot_shape_error_pct": 10.68, "ttft_mape_pct": 72.21, "ttft_shape_error_pct": 27.97}, "aisimulate": {"coverage_pct": 100.0, "points": 33, "status_counts": {"failed": 0, "success": 33, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.62, "tpot_shape_error_pct": 15.23, "ttft_mape_pct": 79.63, "ttft_shape_error_pct": 33.64}, "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.83, "tpot_shape_error_pct": 3.9, "ttft_mape_pct": 81.81, "ttft_shape_error_pct": 13.6}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.4, "tpot_shape_error_pct": 6.38, "ttft_mape_pct": 88.37, "ttft_shape_error_pct": 16.03}, "gpu": "b200", "precisions": ["fp8"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 26.83, "tpot_shape_error_pct": 3.9, "ttft_mape_pct": 81.81, "ttft_shape_error_pct": 13.6}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.4, "tpot_shape_error_pct": 6.38, "ttft_mape_pct": 88.37, "ttft_shape_error_pct": 16.03}, "framework": "sglang", "id": "06e255a2545b66ba", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 10794.947088457546, "interactivity_tok_s": 95.6307040219417, "output_per_gpu": 379.3484367306387, "total_per_gpu": null, "tpot_error_pct": 27.02, "tpot_ms": 10.456892587244344, "tpot_relative": 0.729807, "ttft_error_pct": 79.75, "ttft_ms": 97.54597170658154, "ttft_relative": 0.202508, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9720.495732350011, "interactivity_tok_s": 94.65023205600745, "output_per_gpu": 369.27327610873107, "total_per_gpu": 742.4775916443173, "tpot_error_pct": 26.26, "tpot_ms": 10.565214456191393, "tpot_relative": 0.737367, "ttft_error_pct": 89.28, "ttft_ms": 51.66020694374945, "ttft_relative": 0.107248, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 13601.275306381285, "interactivity_tok_s": 69.79192871072716, "output_per_gpu": 263.91441105492805, "total_per_gpu": 530.6382806390639, "tpot_ms": 14.328304411027087, "tpot_relative": 1.0, "ttft_ms": 481.6890717484057, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13453.901611100473, "interactivity_tok_s": 76.76636568274246, "output_per_gpu": 610.2368982630959, "total_per_gpu": null, "tpot_error_pct": 30.91, "tpot_ms": 13.026538264593214, "tpot_relative": 0.909147, "ttft_error_pct": 80.7, "ttft_ms": 127.75296642161567, "ttft_relative": 0.265219, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12318.61696977496, "interactivity_tok_s": 75.32344280094006, "output_per_gpu": 586.494551482881, "total_per_gpu": 1171.133867326, "tpot_error_pct": 29.59, "tpot_ms": 13.276079302996488, "tpot_relative": 0.926563, "ttft_error_pct": 90.05, "ttft_ms": 65.85743329999977, "ttft_relative": 0.136722, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 18076.22369669407, "interactivity_tok_s": 53.03708896166189, "output_per_gpu": 400.36956723125087, "total_per_gpu": 799.4726608212284, "tpot_ms": 18.854730144085675, "tpot_relative": 1.315908, "ttft_ms": 661.896740739212, "ttft_relative": 1.374116, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18465.653717689696, "interactivity_tok_s": 55.85519003895087, "output_per_gpu": 889.9406373781014, "total_per_gpu": null, "tpot_error_pct": 23.96, "tpot_ms": 17.90343922028813, "tpot_relative": 1.249516, "ttft_error_pct": 81.71, "ttft_ms": 150.43539533493814, "ttft_relative": 0.312308, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17173.254984223448, "interactivity_tok_s": 53.94224814388297, "output_per_gpu": 839.399625510376, "total_per_gpu": 1679.1777172251043, "tpot_error_pct": 21.26, "tpot_ms": 18.538344885675656, "tpot_relative": 1.293827, "ttft_error_pct": 88.55, "ttft_ms": 94.20459645468749, "ttft_relative": 0.195571, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 22524.16227151698, "interactivity_tok_s": 42.474036121389474, "output_per_gpu": 638.2995451109676, "total_per_gpu": 1276.886884972774, "tpot_ms": 23.54379501731437, "tpot_relative": 1.643167, "ttft_ms": 822.546900670568, "ttft_relative": 1.70763, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23118.29435935207, "interactivity_tok_s": 44.54929474362634, "output_per_gpu": 1421.16282212843, "total_per_gpu": null, "tpot_error_pct": 25.44, "tpot_ms": 22.44704446512186, "tpot_relative": 1.566623, "ttft_error_pct": 85.1, "ttft_ms": 154.96787153240763, "ttft_relative": 0.321718, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22161.544534565688, "interactivity_tok_s": 41.77343417024139, "output_per_gpu": 1297.954277957083, "total_per_gpu": 2598.759952709069, "tpot_error_pct": 20.48, "tpot_ms": 23.938659099097514, "tpot_relative": 1.670725, "ttft_error_pct": 85.62, "ttft_ms": 149.53576624296812, "ttft_relative": 0.31044, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 28723.989556438028, "interactivity_tok_s": 33.217997940934715, "output_per_gpu": 1000.2204449333595, "total_per_gpu": 2002.6382133158693, "tpot_ms": 30.104162261016178, "tpot_relative": 2.101028, "ttft_ms": 1040.0363928065417, "ttft_relative": 2.159145, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.28, "tpot_shape_error_pct": 6.08, "ttft_mape_pct": 53.45, "ttft_shape_error_pct": 10.56}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.6, "tpot_shape_error_pct": 3.23, "ttft_mape_pct": 70.55, "ttft_shape_error_pct": 20.32}, "gpu": "b300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.28, "tpot_shape_error_pct": 6.08, "ttft_mape_pct": 53.45, "ttft_shape_error_pct": 10.56}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.6, "tpot_shape_error_pct": 3.23, "ttft_mape_pct": 70.55, "ttft_shape_error_pct": 20.32}, "framework": "sglang", "id": "71859b6d4117a107", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7149.133488286558, "interactivity_tok_s": 144.79130186909714, "output_per_gpu": 142.5952717873778, "total_per_gpu": null, "tpot_error_pct": 2.18, "tpot_ms": 6.906492220810885, "tpot_relative": 1.021783, "ttft_error_pct": 51.13, "ttft_ms": 83.79194639702162, "ttft_relative": 0.488686, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6409.927679725067, "interactivity_tok_s": 144.00811660014526, "output_per_gpu": 139.18015665663225, "total_per_gpu": 279.72144208930666, "tpot_error_pct": 2.73, "tpot_ms": 6.944053040958883, "tpot_relative": 1.02734, "ttft_error_pct": 74.53, "ttft_ms": 43.66505379999972, "ttft_relative": 0.254661, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 6369.864089282055, "interactivity_tok_s": 147.9452909619008, "output_per_gpu": 139.87467319598002, "total_per_gpu": 281.4944933674636, "tpot_ms": 6.759255353774809, "tpot_relative": 1.0, "ttft_ms": 171.46362470230088, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8573.30935606293, "interactivity_tok_s": 120.62025886269197, "output_per_gpu": 238.25482721429475, "total_per_gpu": null, "tpot_error_pct": 1.79, "tpot_ms": 8.290481295835633, "tpot_relative": 1.226538, "ttft_error_pct": 49.53, "ttft_ms": 92.14699042307608, "ttft_relative": 0.537414, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7806.201725549983, "interactivity_tok_s": 119.67971148418906, "output_per_gpu": 233.590414263747, "total_per_gpu": 465.4554412144299, "tpot_error_pct": 2.59, "tpot_ms": 8.35563511641746, "tpot_relative": 1.236177, "ttft_error_pct": 73.36, "ttft_ms": 48.63065046249977, "ttft_relative": 0.283621, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 7746.816220202891, "interactivity_tok_s": 122.77442060486062, "output_per_gpu": 235.09553394544784, "total_per_gpu": 469.03339809265503, "tpot_ms": 8.145019093337185, "tpot_relative": 1.205017, "ttft_ms": 182.5718104999396, "ttft_relative": 1.064785, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10418.883695956793, "interactivity_tok_s": 99.22561059911503, "output_per_gpu": 392.9654259544216, "total_per_gpu": null, "tpot_error_pct": 4.58, "tpot_ms": 10.078043299125023, "tpot_relative": 1.490999, "ttft_error_pct": 48.18, "ttft_ms": 109.04540095189525, "ttft_relative": 0.635968, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9403.511353300026, "interactivity_tok_s": 97.90073857428588, "output_per_gpu": 381.7459018980771, "total_per_gpu": 767.5556185601552, "tpot_error_pct": 3.29, "tpot_ms": 10.214427537144802, "tpot_relative": 1.511176, "ttft_error_pct": 73.7, "ttft_ms": 55.343816912499996, "ttft_relative": 0.322773, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 9892.50284339796, "interactivity_tok_s": 94.68289804712077, "output_per_gpu": 362.81529775584556, "total_per_gpu": 729.2471468300954, "tpot_ms": 10.561569413542145, "tpot_relative": 1.562534, "ttft_ms": 210.4498686916486, "ttft_relative": 1.227373, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13365.508767957503, "interactivity_tok_s": 77.3687746907822, "output_per_gpu": 614.3687200281308, "total_per_gpu": null, "tpot_error_pct": 7.69, "tpot_ms": 12.925110989500277, "tpot_relative": 1.912209, "ttft_error_pct": 50.1, "ttft_ms": 143.1202256987189, "ttft_relative": 0.834697, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12253.81600284061, "interactivity_tok_s": 75.74488777642499, "output_per_gpu": 590.0850251141625, "total_per_gpu": 1178.303456981527, "tpot_error_pct": 5.71, "tpot_ms": 13.20221112415777, "tpot_relative": 1.953205, "ttft_error_pct": 76.1, "ttft_ms": 68.53834272500008, "ttft_relative": 0.399725, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 13221.383392978532, "interactivity_tok_s": 71.41686734434654, "output_per_gpu": 542.7968185823657, "total_per_gpu": 1083.9565510655564, "tpot_ms": 14.002294376458131, "tpot_relative": 2.071574, "ttft_ms": 286.8199783817545, "ttft_relative": 1.672774, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18686.152180665875, "interactivity_tok_s": 55.247661491806085, "output_per_gpu": 879.642055789078, "total_per_gpu": null, "tpot_error_pct": 3.04, "tpot_ms": 18.100313624103574, "tpot_relative": 2.677856, "ttft_error_pct": 49.35, "ttft_ms": 169.5313432079197, "ttft_relative": 0.988731, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17358.74207555011, "interactivity_tok_s": 53.36816048068472, "output_per_gpu": 831.4337529503333, "total_per_gpu": 1663.2423804742143, "tpot_error_pct": 0.37, "tpot_ms": 18.737764071181076, "tpot_relative": 2.772164, "ttft_error_pct": 71.67, "ttft_ms": 94.81108602031252, "ttft_relative": 0.552952, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 17535.992793807964, "interactivity_tok_s": 53.56583451383968, "output_per_gpu": 819.715540002278, "total_per_gpu": 1641.3266363190758, "tpot_ms": 18.668616088518743, "tpot_relative": 2.761934, "ttft_ms": 334.72175415263337, "ttft_relative": 1.952144, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 24227.56624988445, "interactivity_tok_s": 42.53440495656314, "output_per_gpu": 1356.2898610729405, "total_per_gpu": null, "tpot_error_pct": 5.6, "tpot_ms": 23.51037944509197, "tpot_relative": 3.47825, "ttft_error_pct": 55.39, "ttft_ms": 176.44807755536488, "ttft_relative": 1.02907, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 23112.67760831091, "interactivity_tok_s": 40.03941681633269, "output_per_gpu": 1246.2688693264934, "total_per_gpu": 2495.2755909178436, "tpot_error_pct": 0.28, "tpot_ms": 24.97538874222775, "tpot_relative": 3.694991, "ttft_error_pct": 63.05, "ttft_ms": 146.1602914296879, "ttft_relative": 0.852427, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 23306.724383235996, "interactivity_tok_s": 40.15145778473316, "output_per_gpu": 1235.6208727724793, "total_per_gpu": 2475.8619635824543, "tpot_ms": 24.90569596156061, "tpot_relative": 3.68468, "ttft_ms": 395.51614410829643, "ttft_relative": 2.306706, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 32670.32968641587, "interactivity_tok_s": 31.496424574225237, "output_per_gpu": 2010.8591508105837, "total_per_gpu": null, "tpot_error_pct": 5.08, "tpot_ms": 31.749635506829538, "tpot_relative": 4.697209, "ttft_error_pct": 70.48, "ttft_ms": 190.45256292925052, "ttft_relative": 1.110746, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32079.27200002167, "interactivity_tok_s": 28.96955207464832, "output_per_gpu": 1803.2033736972096, "total_per_gpu": 3604.663957182517, "tpot_error_pct": 3.2, "tpot_ms": 34.51900110237171, "tpot_relative": 5.106924, "ttft_error_pct": 61.4, "ttft_ms": 248.9730143144524, "ttft_relative": 1.452046, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 31492.15497038272, "interactivity_tok_s": 29.896161661818827, "output_per_gpu": 1839.0839131584682, "total_per_gpu": 3679.529966390325, "tpot_ms": 33.44910999986752, "tpot_relative": 4.948638, "ttft_ms": 645.0889428452228, "ttft_relative": 3.76225, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 33.18, "tpot_shape_error_pct": 33.67, "ttft_mape_pct": 67.77, "ttft_shape_error_pct": 37.31}, "gpu": "gb200", "precisions": ["fp8"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 66.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 62.39, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "0f53e39038ce97b4", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 56391.46476701867, "interactivity_tok_s": 16.845697098247783, "output_per_gpu": 2716.410234424407, "total_per_gpu": 5434.794463648104, "tpot_error_pct": 66.05, "tpot_ms": 59.36234007816844, "tpot_relative": 1.660473, "ttft_error_pct": 62.39, "ttft_ms": 1756.546504412343, "ttft_relative": 0.376061, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27653706015", "measured": {"e2e_ms": 37573.07614846119, "interactivity_tok_s": 27.971817337929487, "output_per_gpu": 6055.741520950756, "total_per_gpu": 8081.662515575215, "tpot_ms": 35.75026920556966, "tpot_relative": 1.0, "ttft_ms": 4670.90894747007, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.22, "tpot_shape_error_pct": 33.67, "ttft_mape_pct": 69.56, "ttft_shape_error_pct": 37.31}, "framework": "dynamo-sglang", "id": "84ec0fb2a355e028", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 18602.15040318504, "interactivity_tok_s": 51.05693935943023, "output_per_gpu": 1220.5296892823121, "total_per_gpu": 2439.9350521850893, "tpot_error_pct": 0.4, "tpot_ms": 19.585976216870502, "tpot_relative": 0.996002, "ttft_error_pct": 75.62, "ttft_ms": 558.6584341461003, "ttft_relative": 0.243803, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27653706015", "measured": {"e2e_ms": 20406.39200795142, "interactivity_tok_s": 50.8528370246679, "output_per_gpu": 1392.1247279129498, "total_per_gpu": 2227.995813182448, "tpot_ms": 19.66458625533352, "tpot_relative": 1.0, "ttft_ms": 2291.430770064062, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24776.319904980526, "interactivity_tok_s": 38.655294144459994, "output_per_gpu": 1838.0141956492487, "total_per_gpu": 3675.9201425330784, "tpot_error_pct": 17.68, "tpot_ms": 25.869677676306548, "tpot_relative": 1.315547, "ttft_error_pct": 62.25, "ttft_ms": 953.1696124053811, "ttft_relative": 0.415971, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27653706015", "measured": {"e2e_ms": 22766.80035819129, "interactivity_tok_s": 45.48933941114756, "output_per_gpu": 2477.8047258194497, "total_per_gpu": 3966.8655960453625, "tpot_ms": 21.983172605820283, "tpot_relative": 1.117907, "ttft_ms": 2524.7626317070853, "ttft_relative": 1.101828, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 36244.60358590736, "interactivity_tok_s": 26.903958694148457, "output_per_gpu": 2523.9965755643416, "total_per_gpu": 5046.886487546065, "tpot_error_pct": 48.59, "tpot_ms": 37.16925123801567, "tpot_relative": 1.890162, "ttft_error_pct": 70.8, "ttft_ms": 1998.403719160322, "ttft_relative": 0.87212, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27653706015", "measured": {"e2e_ms": 29893.538432544396, "interactivity_tok_s": 39.97572025136113, "output_per_gpu": 3797.7016263383875, "total_per_gpu": 6077.498366262323, "tpot_ms": 25.015184059528007, "tpot_relative": 1.272093, "ttft_ms": 6843.775166259372, "ttft_relative": 2.986682, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 38.14, "tpot_shape_error_pct": 39.11, "ttft_mape_pct": 69.1, "ttft_shape_error_pct": 52.93}, "gpu": "gb300", "precisions": ["fp8"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.73, "tpot_shape_error_pct": null, "ttft_mape_pct": 63.85, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "f1a236144d2e15db", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 52412.88572515476, "interactivity_tok_s": 18.12052709424882, "output_per_gpu": 2922.2147760771636, "total_per_gpu": 5846.553103559517, "tpot_error_pct": 74.73, "tpot_ms": 55.18603265781296, "tpot_relative": 1.747306, "ttft_error_pct": 63.85, "ttft_ms": 1621.6661371010136, "ttft_relative": 0.361489, "unavailable_metrics": {}}, "concurrency": 4096, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29207681289", "measured": {"e2e_ms": 33553.53333613498, "interactivity_tok_s": 31.662100118130475, "output_per_gpu": 6778.241339988579, "total_per_gpu": 9045.871388233823, "tpot_ms": 31.58350192403618, "tpot_relative": 1.0, "ttft_ms": 4486.074840694743, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.94, "tpot_shape_error_pct": 39.11, "ttft_mape_pct": 70.85, "ttft_shape_error_pct": 52.93}, "framework": "dynamo-sglang", "id": "83e3b019f3d0d993", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16985.023723357303, "interactivity_tok_s": 55.92461036931271, "output_per_gpu": 1338.9951140120195, "total_per_gpu": 2676.7567737771064, "tpot_error_pct": 0.23, "tpot_ms": 17.881215325350322, "tpot_relative": 0.997671, "ttft_error_pct": 78.45, "ttft_ms": 512.0262589779293, "ttft_relative": 0.215456, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29207681289", "measured": {"e2e_ms": 18887.191039709953, "interactivity_tok_s": 55.79437779675805, "output_per_gpu": 1483.1345489315256, "total_per_gpu": 2373.6505064166236, "tpot_ms": 17.922952804361326, "tpot_relative": 1.0, "ttft_ms": 2376.476781410895, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24436.321281948043, "interactivity_tok_s": 39.086511831205996, "output_per_gpu": 1869.0814199628178, "total_per_gpu": 3738.0527614743064, "tpot_error_pct": 27.54, "tpot_ms": 25.584273273564854, "tpot_relative": 1.427459, "ttft_error_pct": 62.82, "ttft_ms": 876.1924621154318, "ttft_relative": 0.368694, "unavailable_metrics": {}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29207681289", "measured": {"e2e_ms": 20827.383570417103, "interactivity_tok_s": 49.85062876679864, "output_per_gpu": 2695.4872133030135, "total_per_gpu": 4315.366493417146, "tpot_ms": 20.05992752223853, "tpot_relative": 1.119231, "ttft_ms": 2356.4039657979265, "ttft_relative": 0.991554, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 33682.63294614268, "interactivity_tok_s": 28.909060136659168, "output_per_gpu": 2715.2139756264164, "total_per_gpu": 5429.237447091721, "tpot_error_pct": 50.04, "tpot_ms": 34.59123178936953, "tpot_relative": 1.929996, "ttft_error_pct": 71.28, "ttft_ms": 1811.8527584776518, "ttft_relative": 0.762411, "unavailable_metrics": {}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "29207681289", "measured": {"e2e_ms": 27552.639891269566, "interactivity_tok_s": 43.37489105055948, "output_per_gpu": 4112.900896175727, "total_per_gpu": 6581.914783338902, "tpot_ms": 23.05481295236824, "tpot_relative": 1.286329, "ttft_ms": 6309.13718423516, "ttft_relative": 2.654828, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 20}]}, {"aic": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.1, "tpot_shape_error_pct": 16.17, "ttft_mape_pct": 84.96, "ttft_shape_error_pct": 47.07}, "aisimulate": {"coverage_pct": 100.0, "points": 9, "status_counts": {"failed": 0, "success": 9, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.89, "tpot_shape_error_pct": 19.0, "ttft_mape_pct": 91.58, "ttft_shape_error_pct": 39.74}, "gpu": "h100", "precisions": ["fp8"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.79, "tpot_shape_error_pct": 1.5, "ttft_mape_pct": 76.41, "ttft_shape_error_pct": 13.93}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.43, "tpot_shape_error_pct": 2.07, "ttft_mape_pct": 87.88, "ttft_shape_error_pct": 13.72}, "framework": "sglang", "id": "b02eeec8bef39db1", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5429.43654120125, "interactivity_tok_s": 191.3782527976982, "output_per_gpu": 23.726046687294073, "total_per_gpu": null, "tpot_error_pct": 17.72, "tpot_ms": 5.225254099571482, "tpot_relative": 0.822834, "ttft_error_pct": 76.05, "ttft_ms": 84.00159733962394, "ttft_relative": 0.239508, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 4851.6585187, "interactivity_tok_s": 191.36405205260613, "output_per_gpu": 23.739304725605685, "total_per_gpu": 47.31114094598407, "tpot_error_pct": 17.71, "tpot_ms": 5.225641855269135, "tpot_relative": 0.822895, "ttft_error_pct": 88.03, "ttft_ms": 41.9795116999996, "ttft_relative": 0.119693, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 6195.635022874922, "interactivity_tok_s": 157.47256290408552, "output_per_gpu": 18.58593385918547, "total_per_gpu": 37.04075357675524, "tpot_ms": 6.350312597687807, "tpot_relative": 1.0, "ttft_ms": 350.72518726810813, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 5840.684006423518, "interactivity_tok_s": 177.8179441755669, "output_per_gpu": 44.11001679428711, "total_per_gpu": null, "tpot_error_pct": 15.38, "tpot_ms": 5.6237293971448645, "tpot_relative": 0.885583, "ttft_error_pct": 75.12, "ttft_ms": 87.60883314432185, "ttft_relative": 0.249793, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5298.890229999999, "interactivity_tok_s": 177.79875978578892, "output_per_gpu": 43.99248208095874, "total_per_gpu": 87.18061084079008, "tpot_error_pct": 15.37, "tpot_ms": 5.624336194497617, "tpot_relative": 0.885679, "ttft_error_pct": 87.5, "ttft_ms": 44.009151800000076, "ttft_relative": 0.12548, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 6565.907803922892, "interactivity_tok_s": 150.47296583327397, "output_per_gpu": 35.50702826943969, "total_per_gpu": 70.36485024814704, "tpot_ms": 6.645712035130703, "tpot_relative": 1.046517, "ttft_ms": 352.14359178207815, "ttft_relative": 1.004044, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6890.978679568857, "interactivity_tok_s": 150.46846894980413, "output_per_gpu": 73.91170659862402, "total_per_gpu": null, "tpot_error_pct": 17.61, "tpot_ms": 6.645910648121217, "tpot_relative": 1.046549, "ttft_error_pct": 72.7, "ttft_ms": 92.21208654085248, "ttft_relative": 0.262918, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6194.136714700006, "interactivity_tok_s": 149.16714928373187, "output_per_gpu": 72.00923105580736, "total_per_gpu": 144.72268488938136, "tpot_error_pct": 16.89, "tpot_ms": 6.703888924617665, "tpot_relative": 1.055679, "ttft_error_pct": 85.79, "ttft_ms": 47.992537924999596, "ttft_relative": 0.136838, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 7737.162835965864, "interactivity_tok_s": 123.96756883276929, "output_per_gpu": 57.81681009266138, "total_per_gpu": 116.19904650647739, "tpot_ms": 8.066625887848035, "tpot_relative": 1.270272, "ttft_ms": 337.74865095037967, "ttft_relative": 0.963001, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8627.32388777979, "interactivity_tok_s": 119.98777081854297, "output_per_gpu": 118.34149959448153, "total_per_gpu": null, "tpot_error_pct": 16.45, "tpot_ms": 8.334182668601253, "tpot_relative": 1.312405, "ttft_error_pct": 81.78, "ttft_ms": 101.45501780070923, "ttft_relative": 0.289272, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7857.2583089874925, "interactivity_tok_s": 118.99260019975917, "output_per_gpu": 116.10192355017307, "total_per_gpu": 231.34627429905126, "tpot_error_pct": 15.75, "tpot_ms": 8.403883924893204, "tpot_relative": 1.323381, "ttft_error_pct": 90.19, "ttft_ms": 54.63683412499969, "ttft_relative": 0.155782, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 9819.758574878506, "interactivity_tok_s": 100.25510193785864, "output_per_gpu": 92.84458484942395, "total_per_gpu": 185.0033844139902, "tpot_ms": 9.974554717622574, "tpot_relative": 1.570719, "ttft_ms": 556.8333287010319, "ttft_relative": 1.587663, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.56, "tpot_shape_error_pct": 27.18, "ttft_mape_pct": 91.79, "ttft_shape_error_pct": 71.93}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.06, "tpot_shape_error_pct": 31.69, "ttft_mape_pct": 94.54, "ttft_shape_error_pct": 59.26}, "framework": "sglang", "id": "d286576fb770511f", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12757.795543251166, "interactivity_tok_s": 80.96038422701476, "output_per_gpu": 160.480614125623, "total_per_gpu": null, "tpot_error_pct": 14.82, "tpot_ms": 12.35172003625844, "tpot_relative": 0.851805, "ttft_error_pct": 80.67, "ttft_ms": 121.98594615878214, "ttft_relative": 0.193299, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11495.467188012448, "interactivity_tok_s": 80.06824997165126, "output_per_gpu": 156.25302785357687, "total_per_gpu": 314.16942225111404, "tpot_error_pct": 13.87, "tpot_ms": 12.489345031945337, "tpot_relative": 0.861296, "ttft_error_pct": 89.61, "ttft_ms": 65.56217178124997, "ttft_relative": 0.10389, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 13909.389001509408, "interactivity_tok_s": 68.96247612970313, "output_per_gpu": 129.5571118741794, "total_per_gpu": 260.4934032009681, "tpot_ms": 14.500639421925943, "tpot_relative": 1.0, "ttft_ms": 631.0725222690962, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 15664.607598254212, "interactivity_tok_s": 65.98327001065154, "output_per_gpu": 262.08359507184537, "total_per_gpu": null, "tpot_error_pct": 36.06, "tpot_ms": 15.155356802392063, "tpot_relative": 1.045151, "ttft_error_pct": 94.77, "ttft_ms": 160.67758940713009, "ttft_relative": 0.25461, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14336.19941306253, "interactivity_tok_s": 64.76044292367995, "output_per_gpu": 252.1724890080083, "total_per_gpu": 503.5472904879871, "tpot_error_pct": 34.86, "tpot_ms": 15.441525024442745, "tpot_relative": 1.064886, "ttft_error_pct": 97.24, "ttft_ms": 84.58065403437489, "ttft_relative": 0.134027, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 24918.404718270176, "interactivity_tok_s": 42.18694511107101, "output_per_gpu": 147.41160904975197, "total_per_gpu": 294.35691663066285, "tpot_ms": 23.704015480788453, "tpot_relative": 1.634688, "ttft_ms": 3069.391084340168, "ttft_relative": 4.863769, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21723.625192496023, "interactivity_tok_s": 47.50890530677268, "output_per_gpu": 378.29652545275695, "total_per_gpu": null, "tpot_error_pct": 12.51, "tpot_ms": 21.048685368413317, "tpot_relative": 1.451569, "ttft_error_pct": 91.43, "ttft_ms": 190.82006060919872, "ttft_relative": 0.302374, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20161.789399546877, "interactivity_tok_s": 45.97230659109824, "output_per_gpu": 357.96372820270113, "total_per_gpu": 716.0888541108471, "tpot_error_pct": 16.28, "tpot_ms": 21.752225941032794, "tpot_relative": 1.500087, "ttft_error_pct": 94.56, "ttft_ms": 121.0263536406251, "ttft_relative": 0.191779, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 19440.990141025395, "interactivity_tok_s": 53.454376875955724, "output_per_gpu": 376.7138220071229, "total_per_gpu": 753.5974957106357, "tpot_ms": 18.70754199081889, "tpot_relative": 1.290118, "ttft_ms": 2225.653535636957, "ttft_relative": 3.526779, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28311.51946139209, "interactivity_tok_s": 36.38993612699874, "output_per_gpu": 580.2872227181532, "total_per_gpu": null, "tpot_error_pct": 8.71, "tpot_ms": 27.480125178292667, "tpot_relative": 1.895097, "ttft_error_pct": 94.06, "ttft_ms": 199.35140399868993, "ttft_relative": 0.315893, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27018.581562683412, "interactivity_tok_s": 34.276927466909534, "output_per_gpu": 532.9399145511909, "total_per_gpu": 1067.0506123803714, "tpot_error_pct": 15.41, "tpot_ms": 29.174143480782693, "tpot_relative": 2.011921, "ttft_error_pct": 94.28, "ttft_ms": 191.94450999374993, "ttft_relative": 0.304156, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 26561.879152489564, "interactivity_tok_s": 39.55851431203896, "output_per_gpu": 550.6983822490323, "total_per_gpu": 1102.6065602734404, "tpot_ms": 25.27900800601268, "tpot_relative": 1.743303, "ttft_ms": 3354.2400001471833, "ttft_relative": 5.315142, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 37263.815418189006, "interactivity_tok_s": 27.613603014175975, "output_per_gpu": 881.4887427060859, "total_per_gpu": null, "tpot_error_pct": 5.67, "tpot_ms": 36.214035505856685, "tpot_relative": 2.49741, "ttft_error_pct": 98.05, "ttft_ms": 216.857095697619, "ttft_relative": 0.343633, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 36646.3375062277, "interactivity_tok_s": 25.394209418842273, "output_per_gpu": 788.920203933337, "total_per_gpu": 1577.077919048473, "tpot_error_pct": 14.91, "tpot_ms": 39.37905620554618, "tpot_relative": 2.715677, "ttft_error_pct": 96.98, "ttft_ms": 334.65635789961016, "ttft_relative": 0.530298, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719988201", "measured": {"e2e_ms": 42663.56850901757, "interactivity_tok_s": 29.179308534193257, "output_per_gpu": 667.8201370349136, "total_per_gpu": 1334.9948280734895, "tpot_ms": 34.270860079777684, "tpot_relative": 2.363403, "ttft_ms": 11099.651461491885, "ttft_relative": 17.588551, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.82, "tpot_shape_error_pct": 13.03, "ttft_mape_pct": 67.83, "ttft_shape_error_pct": 31.44}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.69, "tpot_shape_error_pct": 12.14, "ttft_mape_pct": 81.79, "ttft_shape_error_pct": 44.66}, "gpu": "h200", "precisions": ["fp8"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.82, "tpot_shape_error_pct": 13.03, "ttft_mape_pct": 67.83, "ttft_shape_error_pct": 31.44}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 23.69, "tpot_shape_error_pct": 12.14, "ttft_mape_pct": 81.79, "ttft_shape_error_pct": 44.66}, "framework": "sglang", "id": "6592d92ab00a0ba5", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7387.118096369348, "interactivity_tok_s": 140.01328700896488, "output_per_gpu": 69.02632069509197, "total_per_gpu": null, "tpot_error_pct": 16.08, "tpot_ms": 7.142179298568793, "tpot_relative": 0.839229, "ttft_error_pct": 74.29, "ttft_ms": 80.66867393347222, "ttft_relative": 0.25707, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6636.030138825001, "interactivity_tok_s": 139.03453454830952, "output_per_gpu": 67.13974711455595, "total_per_gpu": 134.9360953136995, "tpot_error_pct": 15.49, "tpot_ms": 7.192457638303783, "tpot_relative": 0.845137, "ttft_error_pct": 86.58, "ttft_ms": 42.10919197499965, "ttft_relative": 0.134191, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 8118.556019570679, "interactivity_tok_s": 117.50318032801614, "output_per_gpu": 54.9858896249818, "total_per_gpu": 110.50952025013709, "tpot_ms": 8.510407949882282, "tpot_relative": 1.0, "ttft_ms": 313.800502801314, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8317.877587273935, "interactivity_tok_s": 124.31442747532068, "output_per_gpu": 122.78891779383912, "total_per_gpu": null, "tpot_error_pct": 19.73, "tpot_ms": 8.044118613654263, "tpot_relative": 0.94521, "ttft_error_pct": 72.84, "ttft_ms": 88.74424550562502, "ttft_relative": 0.282805, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7583.600053987508, "interactivity_tok_s": 123.20510639444848, "output_per_gpu": 120.01229160239227, "total_per_gpu": 239.13812694331864, "tpot_error_pct": 19.0, "tpot_ms": 8.116546702199505, "tpot_relative": 0.95372, "ttft_error_pct": 85.16, "ttft_ms": 48.49349069999996, "ttft_relative": 0.154536, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 9637.635974003933, "interactivity_tok_s": 99.79086824560643, "output_per_gpu": 94.62224388289614, "total_per_gpu": 188.54557201771343, "tpot_ms": 10.020957003187792, "tpot_relative": 1.177494, "ttft_ms": 326.78975421004, "ttft_relative": 1.041393, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 11464.677154968025, "interactivity_tok_s": 90.05558748687594, "output_per_gpu": 178.59106689054866, "total_per_gpu": null, "tpot_error_pct": 22.01, "tpot_ms": 11.104252694434233, "tpot_relative": 1.304785, "ttft_error_pct": 70.37, "ttft_ms": 105.02664856180576, "ttft_relative": 0.334692, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 10313.035558418782, "interactivity_tok_s": 89.23633607177881, "output_per_gpu": 174.3029050378919, "total_per_gpu": 350.46132369198597, "tpot_error_pct": 21.3, "tpot_ms": 11.206197430558248, "tpot_relative": 1.316764, "ttft_error_pct": 83.9, "ttft_ms": 57.092386362499646, "ttft_relative": 0.181938, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 13397.311967390124, "interactivity_tok_s": 70.2299979688885, "output_per_gpu": 134.58238083943382, "total_per_gpu": 270.59743682615897, "tpot_ms": 14.238929644323706, "tpot_relative": 1.67312, "ttft_ms": 354.50660856440663, "ttft_relative": 1.12972, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14170.422309724683, "interactivity_tok_s": 72.90345520826742, "output_per_gpu": 289.69978837015645, "total_per_gpu": null, "tpot_error_pct": 31.41, "tpot_ms": 13.716770997248943, "tpot_relative": 1.611764, "ttft_error_pct": 62.01, "ttft_ms": 138.16557953901454, "ttft_relative": 0.440298, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12975.73223773129, "interactivity_tok_s": 71.53506783663951, "output_per_gpu": 278.67502792532895, "total_per_gpu": 556.4685338613887, "tpot_error_pct": 30.09, "tpot_ms": 13.979157778722486, "tpot_relative": 1.642595, "ttft_error_pct": 79.63, "ttft_ms": 74.08606497187525, "ttft_relative": 0.236093, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 18843.7076468952, "interactivity_tok_s": 50.00722339479257, "output_per_gpu": 192.88030123651566, "total_per_gpu": 385.1504716403454, "tpot_ms": 19.997111059442137, "tpot_relative": 2.349724, "ttft_ms": 363.68632859084755, "ttft_relative": 1.158973, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19318.942575990623, "interactivity_tok_s": 53.40529647345653, "output_per_gpu": 425.35037677720766, "total_per_gpu": null, "tpot_error_pct": 34.89, "tpot_ms": 18.72473454944716, "tpot_relative": 2.200216, "ttft_error_pct": 59.61, "ttft_ms": 163.5391319061765, "ttft_relative": 0.521156, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17969.270156962524, "interactivity_tok_s": 51.57733567927016, "output_per_gpu": 401.39125487966686, "total_per_gpu": 802.9634879490751, "tpot_error_pct": 32.59, "tpot_ms": 19.388360930824845, "tpot_relative": 2.278194, "ttft_error_pct": 73.67, "ttft_ms": 106.62348293437523, "ttft_relative": 0.339781, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 26948.797989080776, "interactivity_tok_s": 34.77037909622568, "output_per_gpu": 269.25788273274964, "total_per_gpu": 538.6371677222703, "tpot_ms": 28.760112083694533, "tpot_relative": 3.379405, "ttft_ms": 404.9074711392677, "ttft_relative": 1.290334, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 33}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.81, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 80.04, "ttft_shape_error_pct": 6.39}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.62, "tpot_shape_error_pct": 4.53, "ttft_mape_pct": 87.69, "ttft_shape_error_pct": 26.62}, "gpu_skus": ["b200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.81, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 80.04, "ttft_shape_error_pct": 6.39}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.62, "tpot_shape_error_pct": 4.53, "ttft_mape_pct": 87.69, "ttft_shape_error_pct": 26.62}, "gpu": "b200", "precisions": ["fp8"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.81, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 80.04, "ttft_shape_error_pct": 6.39}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.62, "tpot_shape_error_pct": 4.53, "ttft_mape_pct": 87.69, "ttft_shape_error_pct": 26.62}, "framework": "sglang", "id": "c4012f36bbac1852", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 86170.71001105398, "interactivity_tok_s": 95.164250228265, "output_per_gpu": 380.25735278506994, "total_per_gpu": null, "tpot_error_pct": 16.39, "tpot_ms": 10.508147729860296, "tpot_relative": 0.83606, "ttft_error_pct": 80.38, "ttft_ms": 98.47195576829829, "ttft_relative": 0.196198, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 77097.0398925439, "interactivity_tok_s": 95.74585763430444, "output_per_gpu": 373.7653176065081, "total_per_gpu": 420.6580666615306, "tpot_error_pct": 16.9, "tpot_ms": 10.44431607495167, "tpot_relative": 0.830981, "ttft_error_pct": 89.73, "ttft_ms": 51.522017099999104, "ttft_relative": 0.102654, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 93237.2050573671, "interactivity_tok_s": 79.5630348748346, "output_per_gpu": 308.9846266942207, "total_per_gpu": 347.7499638694739, "tpot_ms": 12.568650775742281, "tpot_relative": 1.0, "ttft_ms": 501.8999164283742, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 107234.47349690773, "interactivity_tok_s": 76.47671277482519, "output_per_gpu": 611.3178784226645, "total_per_gpu": null, "tpot_error_pct": 17.21, "tpot_ms": 13.075875828297667, "tpot_relative": 1.040356, "ttft_error_pct": 77.89, "ttft_ms": 129.9745873215463, "ttft_relative": 0.258965, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 96127.08968435842, "interactivity_tok_s": 77.18920831362743, "output_per_gpu": 603.2889344284453, "total_per_gpu": 678.1934253576953, "tpot_error_pct": 17.97, "tpot_ms": 12.955178863046509, "tpot_relative": 1.030753, "ttft_error_pct": 88.99, "ttft_ms": 64.6993388625013, "ttft_relative": 0.128909, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 117723.86026121385, "interactivity_tok_s": 63.3173969241536, "output_per_gpu": 492.4425195507203, "total_per_gpu": 553.5842944679246, "tpot_ms": 15.793447750195355, "tpot_relative": 1.256575, "ttft_ms": 587.7308338873263, "ttft_relative": 1.171012, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 151247.37208012812, "interactivity_tok_s": 54.21254014622016, "output_per_gpu": 866.9372555199297, "total_per_gpu": null, "tpot_error_pct": 9.57, "tpot_ms": 18.445916706777346, "tpot_relative": 1.467613, "ttft_error_pct": 80.32, "ttft_ms": 156.86833491486422, "ttft_relative": 0.312549, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 134428.56132956428, "interactivity_tok_s": 54.97499549089536, "output_per_gpu": 858.2513794043662, "total_per_gpu": 965.411838783387, "tpot_error_pct": 10.82, "tpot_ms": 18.190087894879667, "tpot_relative": 1.447259, "ttft_error_pct": 88.43, "ttft_ms": 92.25786844999905, "ttft_relative": 0.183817, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 151472.967937258, "interactivity_tok_s": 49.02537206676245, "output_per_gpu": 760.4129863167349, "total_per_gpu": 855.3574360279914, "tpot_ms": 20.397601442742875, "tpot_relative": 1.622895, "ttft_ms": 797.130254309377, "ttft_relative": 1.588226, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 186432.6767701262, "interactivity_tok_s": 43.97412849210505, "output_per_gpu": 1406.586386823947, "total_per_gpu": null, "tpot_error_pct": 12.09, "tpot_ms": 22.740643971591982, "tpot_relative": 1.809315, "ttft_error_pct": 81.59, "ttft_ms": 164.06199881630798, "ttft_relative": 0.326882, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 166363.94734664302, "interactivity_tok_s": 44.31654080274774, "output_per_gpu": 1382.4825747073676, "total_per_gpu": 1555.5228898725943, "tpot_error_pct": 12.77, "tpot_ms": 22.564938099545834, "tpot_relative": 1.795335, "ttft_error_pct": 83.59, "ttft_ms": 146.25729142968896, "ttft_relative": 0.291407, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 191487.33258958746, "interactivity_tok_s": 38.65773645442662, "output_per_gpu": 1201.0608908431336, "total_per_gpu": 1351.3933137513325, "tpot_ms": 25.868043287503244, "tpot_relative": 2.05814, "ttft_ms": 891.160001078606, "ttft_relative": 1.775573, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}], "identity": "1024:8192", "label": "1k8k", "precisions": ["fp8"], "rows": 4}, {"aic": {"coverage_pct": 89.09, "points": 49, "status_counts": {"failed": 6, "success": 49, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.05, "tpot_shape_error_pct": 4.0, "ttft_mape_pct": 44.9, "ttft_shape_error_pct": 42.94}, "aisimulate": {"coverage_pct": 60.0, "points": 33, "status_counts": {"failed": 22, "success": 33, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.98, "tpot_shape_error_pct": 5.77, "ttft_mape_pct": 43.16, "ttft_shape_error_pct": 19.17}, "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "gpus": [{"aic": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.1, "tpot_shape_error_pct": 4.71, "ttft_mape_pct": 58.37, "ttft_shape_error_pct": 47.64}, "aisimulate": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.32, "tpot_shape_error_pct": 6.44, "ttft_mape_pct": 39.05, "ttft_shape_error_pct": 18.1}, "gpu": "b200", "precisions": ["fp8"], "rows": 17, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 37.82, "tpot_shape_error_pct": 4.6, "ttft_mape_pct": 50.7, "ttft_shape_error_pct": 61.51}, "aisimulate": {"points": 10, "status_counts": {"failed": 0, "success": 10, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 40.13, "tpot_shape_error_pct": 6.98, "ttft_mape_pct": 34.2, "ttft_shape_error_pct": 23.66}, "framework": "sglang", "id": "8d0f27f9cecb97cb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7250.224683452627, "interactivity_tok_s": 147.68718659250382, "output_per_gpu": 72.11680688037164, "total_per_gpu": null, "tpot_error_pct": 34.86, "tpot_ms": 6.771068114115982, "tpot_relative": 1.348616, "ttft_error_pct": 73.05, "ttft_ms": 323.42200271197737, "ttft_relative": 1.730479, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6403.541058199968, "interactivity_tok_s": 149.56907427139552, "output_per_gpu": 72.80028480094796, "total_per_gpu": 648.7573735998511, "tpot_error_pct": 33.16, "tpot_ms": 6.68587410112256, "tpot_relative": 1.331647, "ttft_error_pct": 16.4, "ttft_ms": 156.25530059999952, "ttft_relative": 0.836049, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 4878.639742103405, "interactivity_tok_s": 199.1732647623482, "output_per_gpu": 95.54692317030822, "total_per_gpu": 851.463302664288, "tpot_ms": 5.0207541719677655, "tpot_relative": 1.0, "ttft_ms": 186.89734919462353, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 8550.479935487083, "interactivity_tok_s": 124.59758085335724, "output_per_gpu": 117.6942783991818, "total_per_gpu": null, "tpot_error_pct": 36.28, "tpot_ms": 8.025838007055137, "tpot_relative": 1.598532, "ttft_error_pct": 59.55, "ttft_ms": 340.0476542696777, "ttft_relative": 1.819435, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7717.979076974991, "interactivity_tok_s": 121.78173823144827, "output_per_gpu": 116.55432978081025, "total_per_gpu": 1047.75391331694, "tpot_error_pct": 39.43, "tpot_ms": 8.211411780799867, "tpot_relative": 1.635494, "ttft_error_pct": 20.08, "ttft_ms": 170.33675564999996, "ttft_relative": 0.911392, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 5625.119100650772, "interactivity_tok_s": 169.8012297141015, "output_per_gpu": 159.8357502328458, "total_per_gpu": 1436.828070731917, "tpot_ms": 5.889238857007835, "tpot_relative": 1.172979, "ttft_ms": 213.13314277213067, "ttft_relative": 1.140375, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10230.356410642435, "interactivity_tok_s": 103.78388170596183, "output_per_gpu": 197.9579100140499, "total_per_gpu": null, "tpot_error_pct": 33.09, "tpot_ms": 9.635407575457407, "tpot_relative": 1.919116, "ttft_error_pct": 7.11, "ttft_ms": 373.3344609495073, "ttft_relative": 1.997537, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9275.364800924977, "interactivity_tok_s": 102.05485369440146, "output_per_gpu": 196.51521713279126, "total_per_gpu": 1748.0937130786056, "tpot_error_pct": 35.34, "tpot_ms": 9.79865203662389, "tpot_relative": 1.95163, "ttft_error_pct": 41.95, "ttft_ms": 202.34972047500008, "ttft_relative": 1.082678, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 7050.228059012443, "interactivity_tok_s": 138.12392081713207, "output_per_gpu": 258.4755695735468, "total_per_gpu": 2299.259694737061, "tpot_ms": 7.239875570314436, "tpot_relative": 1.44199, "ttft_ms": 348.5680403362494, "ttft_relative": 1.865024, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13475.049099003601, "interactivity_tok_s": 78.48308944254836, "output_per_gpu": 302.93762544237205, "total_per_gpu": null, "tpot_error_pct": 28.29, "tpot_ms": 12.741598312487758, "tpot_relative": 2.537786, "ttft_error_pct": 2.26, "ttft_ms": 440.3940253286254, "ttft_relative": 2.356342, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11991.717054981165, "interactivity_tok_s": 78.01144914585765, "output_per_gpu": 296.920687568446, "total_per_gpu": 2679.634886924614, "tpot_error_pct": 29.07, "tpot_ms": 12.818631251552635, "tpot_relative": 2.553129, "ttft_error_pct": 41.65, "ttft_ms": 262.9286096187502, "ttft_relative": 1.406808, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 9540.56237572513, "interactivity_tok_s": 100.68638151210698, "output_per_gpu": 373.42344955234176, "total_per_gpu": 3370.0531654114598, "tpot_ms": 9.931829756735826, "tpot_relative": 1.978155, "ttft_ms": 450.5728443269618, "ttft_relative": 2.410804, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19064.577510152612, "interactivity_tok_s": 55.3337143390951, "output_per_gpu": 431.8003315943191, "total_per_gpu": null, "tpot_error_pct": 30.05, "tpot_ms": 18.072164718092434, "tpot_relative": 3.599492, "ttft_error_pct": 16.05, "ttft_ms": 576.7530035440506, "ttft_relative": 3.085935, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17132.931391696948, "interactivity_tok_s": 55.23923185574642, "output_per_gpu": 422.79621717775365, "total_per_gpu": 3780.778395272598, "tpot_error_pct": 30.28, "tpot_ms": 18.103075774323464, "tpot_relative": 3.605649, "ttft_error_pct": 45.13, "ttft_ms": 376.94823385624954, "ttft_relative": 2.016873, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 13549.064003957028, "interactivity_tok_s": 71.96331189690977, "output_per_gpu": 533.7630146174243, "total_per_gpu": 4773.078830581632, "tpot_ms": 13.895969677334177, "tpot_relative": 2.767706, "ttft_ms": 686.9940978314844, "ttft_relative": 3.675783, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29823.938110926458, "interactivity_tok_s": 35.098901791101326, "output_per_gpu": 553.0192227295681, "total_per_gpu": null, "tpot_error_pct": 40.87, "tpot_ms": 28.490919914010856, "tpot_relative": 5.67463, "ttft_error_pct": 38.82, "ttft_ms": 677.7270388933516, "ttft_relative": 3.626199, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26975.044019121728, "interactivity_tok_s": 34.968724468493704, "output_per_gpu": 538.6320281939496, "total_per_gpu": 4855.109512859234, "tpot_error_pct": 41.39, "tpot_ms": 28.596982452161928, "tpot_relative": 5.695754, "ttft_error_pct": 45.24, "ttft_ms": 606.6629823078134, "ttft_relative": 3.245969, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 19757.913123023172, "interactivity_tok_s": 49.44213827235092, "output_per_gpu": 733.3303737062694, "total_per_gpu": 6610.07717158606, "tpot_ms": 20.225662460056284, "tpot_relative": 4.028411, "ttft_ms": 1107.7683070448984, "ttft_relative": 5.927148, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44724.63043600244, "interactivity_tok_s": 23.234440837418706, "output_per_gpu": 736.2234727421795, "total_per_gpu": null, "tpot_error_pct": 37.8, "tpot_ms": 43.0395552446227, "tpot_relative": 8.572329, "ttft_error_pct": 60.81, "ttft_ms": 695.1654207534141, "ttft_relative": 3.719504, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41014.63873151811, "interactivity_tok_s": 23.047453353298877, "output_per_gpu": 708.4325302278713, "total_per_gpu": 6395.786864676206, "tpot_error_pct": 38.92, "tpot_ms": 43.38874168311814, "tpot_relative": 8.641877, "ttft_error_pct": 40.15, "ttft_ms": 1061.5621189492203, "ttft_relative": 5.679921, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 30535.396918332663, "interactivity_tok_s": 32.0164110713556, "output_per_gpu": 950.4361105651406, "total_per_gpu": 8580.614994784602, "tpot_ms": 31.233981778010047, "tpot_relative": 6.220974, "ttft_ms": 1773.7798382891924, "ttft_relative": 9.490663, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 73739.45340437129, "interactivity_tok_s": 14.00955388377153, "output_per_gpu": 891.7797680199728, "total_per_gpu": null, "tpot_error_pct": 40.87, "tpot_ms": 71.37986036503175, "tpot_relative": 14.21696, "ttft_error_pct": 76.01, "ttft_ms": 717.8562509438169, "ttft_relative": 3.840912, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 69000.12651201007, "interactivity_tok_s": 13.776285758483246, "output_per_gpu": 846.6060982974236, "total_per_gpu": 7617.264174733555, "tpot_error_pct": 43.25, "tpot_ms": 72.58850589566305, "tpot_relative": 14.45769, "ttft_error_pct": 33.84, "ttft_ms": 1980.0859898101637, "ttft_relative": 10.594511, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 49781.7756787239, "interactivity_tok_s": 19.734629680031475, "output_per_gpu": 1171.952939158891, "total_per_gpu": 10544.543862702627, "tpot_ms": 50.672346844787874, "tpot_relative": 10.092577, "ttft_ms": 2992.8493169101785, "ttft_relative": 16.013332, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 130118.560564506, "interactivity_tok_s": 7.909405159245607, "output_per_gpu": 1009.8173158092743, "total_per_gpu": null, "tpot_error_pct": 47.62, "tpot_ms": 126.43175812419491, "tpot_relative": 25.181826, "ttft_error_pct": 85.61, "ttft_ms": 778.872003454603, "ttft_relative": 4.167379, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 125333.70772200846, "interactivity_tok_s": 7.6007253453170565, "output_per_gpu": 934.2412737186606, "total_per_gpu": 8408.239335238903, "tpot_error_pct": 53.62, "tpot_ms": 131.56639064929743, "tpot_relative": 26.204508, "ttft_error_pct": 29.55, "ttft_ms": 3814.5775640211336, "ttft_relative": 20.410014, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 84551.33646101494, "interactivity_tok_s": 11.675999383410561, "output_per_gpu": 1383.4720624066501, "total_per_gpu": 12451.349069635238, "tpot_ms": 85.64577362181222, "tpot_relative": 17.058348, "ttft_ms": 5414.289765332251, "ttft_relative": 28.969323, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 157185.35787291397, "interactivity_tok_s": 6.541939634569677, "output_per_gpu": 1044.6956597558687, "total_per_gpu": null, "tpot_error_pct": 48.47, "tpot_ms": 152.85986356640834, "tpot_relative": 30.445598, "ttft_error_pct": 87.69, "ttft_ms": 809.7174444782361, "ttft_relative": 4.332418, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 153841.26984649396, "interactivity_tok_s": 6.191200096527488, "output_per_gpu": 951.2954448484187, "total_per_gpu": 8568.322175724325, "tpot_error_pct": 56.89, "tpot_ms": 161.51957365436772, "tpot_relative": 32.170381, "ttft_error_pct": 28.0, "ttft_ms": 4737.245337108612, "ttft_relative": 25.346776, "unavailable_metrics": {}}, "concurrency": 640, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 101671.64695873536, "interactivity_tok_s": 9.713108401958143, "output_per_gpu": 1437.8960905379176, "total_per_gpu": 12951.136290688852, "tpot_ms": 102.95365382706962, "tpot_relative": 20.505615, "ttft_ms": 6579.116053285543, "ttft_relative": 35.201762, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 10, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.51, "tpot_shape_error_pct": 6.64, "ttft_mape_pct": 63.84, "ttft_shape_error_pct": 30.73}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.15, "tpot_shape_error_pct": 6.35, "ttft_mape_pct": 70.28, "ttft_shape_error_pct": 8.13}, "framework": "sglang", "id": "ffdab2a53e7c5657", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 4, "moe_tp_size": 1, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 13783.178898515158, "interactivity_tok_s": 76.78236662658416, "output_per_gpu": 296.1384646672775, "total_per_gpu": null, "tpot_error_pct": 20.47, "tpot_ms": 13.02382361907783, "tpot_relative": 0.795268, "ttft_error_pct": 53.01, "ttft_ms": 459.80733619853896, "ttft_relative": 0.469939, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12267.183259643836, "interactivity_tok_s": 76.29368177145568, "output_per_gpu": 290.30438211144434, "total_per_gpu": 2619.9243862170997, "tpot_error_pct": 19.96, "tpot_ms": 13.107245276163056, "tpot_relative": 0.800362, "ttft_error_pct": 71.99, "ttft_ms": 274.10350276250006, "ttft_relative": 0.280143, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 15976.711735050776, "interactivity_tok_s": 61.06253027124691, "output_per_gpu": 223.158965119909, "total_per_gpu": 2013.953804170193, "tpot_ms": 16.376655136265775, "tpot_relative": 1.0, "ttft_ms": 978.4397798175632, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19474.63512490216, "interactivity_tok_s": 54.205792918000675, "output_per_gpu": 422.74469438909995, "total_per_gpu": null, "tpot_error_pct": 18.8, "tpot_ms": 18.44821274937793, "tpot_relative": 1.126495, "ttft_error_pct": 56.56, "ttft_ms": 602.1134822885388, "ttft_relative": 0.615381, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17498.335938346863, "interactivity_tok_s": 54.11650039938306, "output_per_gpu": 414.05563477411414, "total_per_gpu": 3702.6173243567564, "tpot_error_pct": 18.67, "tpot_ms": 18.47865240028345, "tpot_relative": 1.128353, "ttft_error_pct": 71.56, "ttft_ms": 394.14854469999983, "ttft_relative": 0.402834, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 22435.419576311142, "interactivity_tok_s": 44.01247876056922, "output_per_gpu": 322.70437103607185, "total_per_gpu": 2885.725237130645, "tpot_ms": 22.72082891400109, "tpot_relative": 1.387391, "ttft_ms": 1385.983195361041, "ttft_relative": 1.416524, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30514.136214736805, "interactivity_tok_s": 34.321021617302634, "output_per_gpu": 540.5721368158268, "total_per_gpu": null, "tpot_error_pct": 12.16, "tpot_ms": 29.136661814747935, "tpot_relative": 1.779158, "ttft_error_pct": 67.53, "ttft_ms": 707.3311782496667, "ttft_relative": 0.722917, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27584.346831089224, "interactivity_tok_s": 34.2154446495892, "output_per_gpu": 526.7963547404136, "total_per_gpu": 4748.425380153581, "tpot_error_pct": 11.89, "tpot_ms": 29.226567424194098, "tpot_relative": 1.784648, "ttft_error_pct": 70.88, "ttft_ms": 634.4501020031252, "ttft_relative": 0.64843, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 32779.24590507682, "interactivity_tok_s": 30.14718222196062, "output_per_gpu": 442.8306970442271, "total_per_gpu": 3991.5775840781375, "tpot_ms": 33.17059593289462, "tpot_relative": 2.025481, "ttft_ms": 2178.391830824694, "ttft_relative": 2.226393, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 45607.616342470894, "interactivity_tok_s": 22.792875714208858, "output_per_gpu": 722.0341317084811, "total_per_gpu": null, "tpot_error_pct": 14.6, "tpot_ms": 43.873358172905306, "tpot_relative": 2.679018, "ttft_error_pct": 78.25, "ttft_ms": 725.1709315887636, "ttft_relative": 0.74115, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 41754.03587352647, "interactivity_tok_s": 22.65721539412014, "output_per_gpu": 695.9769901153429, "total_per_gpu": 6283.337229114535, "tpot_error_pct": 14.09, "tpot_ms": 44.13605037534815, "tpot_relative": 2.695059, "ttft_error_pct": 66.69, "ttft_ms": 1110.7300542375003, "ttft_relative": 1.135205, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "23467712427", "measured": {"e2e_ms": 50659.37893614282, "interactivity_tok_s": 19.46497527610712, "output_per_gpu": 573.4564819834521, "total_per_gpu": 5177.212053988334, "tpot_ms": 51.374326749208905, "tpot_relative": 3.137046, "ttft_ms": 3334.467003904683, "ttft_relative": 3.407943, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}, {"aic": {"coverage_pct": 100.0, "points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 28.15, "tpot_shape_error_pct": 2.26, "ttft_mape_pct": 76.69, "ttft_shape_error_pct": 10.55}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.85, "tpot_shape_error_pct": 4.17, "ttft_mape_pct": 13.59, "ttft_shape_error_pct": 8.03}, "framework": "sglang", "id": "746e7781afba05fb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5957.1451591316145, "interactivity_tok_s": 179.19250305024013, "output_per_gpu": 21.89799290952668, "total_per_gpu": null, "tpot_error_pct": 26.25, "tpot_ms": 5.5805906105325755, "tpot_relative": 1.262492, "ttft_error_pct": 90.05, "ttft_ms": 248.20096455678984, "ttft_relative": 1.900497, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5229.260328200004, "interactivity_tok_s": 181.86702987637267, "output_per_gpu": 22.237848701639997, "total_per_gpu": 199.06256997503735, "tpot_error_pct": 24.39, "tpot_ms": 5.498522743125941, "tpot_relative": 1.243925, "ttft_error_pct": 8.71, "ttft_ms": 119.22868439999884, "ttft_relative": 0.912945, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 4238.176314951852, "interactivity_tok_s": 226.2290248715852, "output_per_gpu": 27.43617537827889, "total_per_gpu": 245.5955004623834, "tpot_ms": 4.420299298764302, "tpot_relative": 1.0, "ttft_ms": 130.59790758416057, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6311.029927760311, "interactivity_tok_s": 168.99026993974604, "output_per_gpu": 41.345917229904884, "total_per_gpu": null, "tpot_error_pct": 31.46, "tpot_ms": 5.917500459384749, "tpot_relative": 1.33871, "ttft_error_pct": 71.84, "ttft_ms": 257.42695780971206, "ttft_relative": 1.971142, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5574.537969399996, "interactivity_tok_s": 171.45535674454027, "output_per_gpu": 41.81431317115306, "total_per_gpu": 372.62689378168045, "tpot_error_pct": 29.57, "tpot_ms": 5.832422030942719, "tpot_relative": 1.319463, "ttft_error_pct": 16.65, "ttft_ms": 124.86924579999996, "ttft_relative": 0.956135, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 4356.094122887589, "interactivity_tok_s": 222.15528249733217, "output_per_gpu": 53.49488790463337, "total_per_gpu": 476.71795615800227, "tpot_ms": 4.501355937876512, "tpot_relative": 1.018337, "ttft_ms": 149.807350942865, "ttft_relative": 1.147088, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6695.788497708024, "interactivity_tok_s": 159.22047347466864, "output_per_gpu": 75.12458305435483, "total_per_gpu": null, "tpot_error_pct": 26.74, "tpot_ms": 6.280599336109223, "tpot_relative": 1.420854, "ttft_error_pct": 68.18, "ttft_ms": 270.7353768682885, "ttft_relative": 2.073045, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6038.814455500017, "interactivity_tok_s": 155.72408110021553, "output_per_gpu": 74.33097256432445, "total_per_gpu": 668.1911133750732, "tpot_error_pct": 29.59, "tpot_ms": 6.421614389597551, "tpot_relative": 1.452756, "ttft_error_pct": 15.42, "ttft_ms": 136.16389110000017, "ttft_relative": 1.042619, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35317697106", "measured": {"e2e_ms": 4715.558852942195, "interactivity_tok_s": 201.79975480143545, "output_per_gpu": 95.11296447592474, "total_per_gpu": 855.0088265638385, "tpot_ms": 4.955407408616369, "tpot_relative": 1.121057, "ttft_ms": 160.98307562060654, "ttft_relative": 1.232662, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.34, "tpot_shape_error_pct": 3.25, "ttft_mape_pct": 28.04, "ttft_shape_error_pct": 39.69}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.62, "tpot_shape_error_pct": 3.36, "ttft_mape_pct": 36.97, "ttft_shape_error_pct": 20.28}, "gpu": "b300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 10.34, "tpot_shape_error_pct": 3.25, "ttft_mape_pct": 28.04, "ttft_shape_error_pct": 39.69}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.62, "tpot_shape_error_pct": 3.36, "ttft_mape_pct": 36.97, "ttft_shape_error_pct": 20.28}, "framework": "sglang", "id": "4c0380df4059a175", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 8210.558146082742, "interactivity_tok_s": 129.83848313454106, "output_per_gpu": 122.53266699957918, "total_per_gpu": null, "tpot_error_pct": 10.34, "tpot_ms": 7.701876792289551, "tpot_relative": 1.103368, "ttft_error_pct": 15.96, "ttft_ms": 331.5381875705301, "ttft_relative": 1.159593, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7411.367222824954, "interactivity_tok_s": 126.86371758433052, "output_per_gpu": 121.38628290016534, "total_per_gpu": 1091.190290149007, "tpot_error_pct": 12.92, "tpot_ms": 7.882474351543946, "tpot_relative": 1.129241, "ttft_error_pct": 41.92, "ttft_ms": 166.0681261749998, "ttft_relative": 0.580842, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 6701.099075021921, "interactivity_tok_s": 143.2596657992751, "output_per_gpu": 134.13846582473047, "total_per_gpu": 1206.1564654484785, "tpot_ms": 6.980331794164076, "tpot_relative": 1.0, "ttft_ms": 285.9091628633905, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 10164.186789882539, "interactivity_tok_s": 104.3863000093697, "output_per_gpu": 199.29133232542034, "total_per_gpu": null, "tpot_error_pct": 9.39, "tpot_ms": 9.579801179946411, "tpot_relative": 1.372399, "ttft_error_pct": 10.17, "ttft_ms": 364.05018279736026, "ttft_relative": 1.273307, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9208.010170975027, "interactivity_tok_s": 102.75988023284958, "output_per_gpu": 198.04219550801412, "total_per_gpu": 1761.6768917081279, "tpot_error_pct": 11.12, "tpot_ms": 9.73142434317792, "tpot_relative": 1.394121, "ttft_error_pct": 51.36, "ttft_ms": 197.10393726249976, "ttft_relative": 0.689394, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 8513.134269900911, "interactivity_tok_s": 114.18697175541993, "output_per_gpu": 210.76791356263175, "total_per_gpu": 1875.3873929558572, "tpot_ms": 8.757566512420754, "tpot_relative": 1.254606, "ttft_ms": 405.2475653224974, "ttft_relative": 1.4174, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13115.928290192147, "interactivity_tok_s": 80.636778587421, "output_per_gpu": 311.22982499860683, "total_per_gpu": null, "tpot_error_pct": 5.18, "tpot_ms": 12.401289058389985, "tpot_relative": 1.776605, "ttft_error_pct": 16.95, "ttft_ms": 429.4095834591929, "ttft_relative": 1.501909, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11674.643569387614, "interactivity_tok_s": 80.13114836854449, "output_per_gpu": 305.0555669556065, "total_per_gpu": 2753.0501372571935, "tpot_error_pct": 5.84, "tpot_ms": 12.479541605977413, "tpot_relative": 1.787815, "ttft_error_pct": 50.47, "ttft_ms": 256.07173091249973, "ttft_relative": 0.89564, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 11310.692766893771, "interactivity_tok_s": 84.81297423512986, "output_per_gpu": 311.0107410071067, "total_per_gpu": 2807.5842253795354, "tpot_ms": 11.790648883833109, "tpot_relative": 1.689124, "ttft_ms": 517.0539377300884, "ttft_relative": 1.808455, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18853.808318894164, "interactivity_tok_s": 55.927870069172656, "output_per_gpu": 436.6035856235289, "total_per_gpu": null, "tpot_error_pct": 5.7, "tpot_ms": 17.8801731366344, "tpot_relative": 2.561508, "ttft_error_pct": 6.57, "ttft_ms": 562.391200117171, "ttft_relative": 1.967028, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16945.217623099976, "interactivity_tok_s": 55.83380456288685, "output_per_gpu": 427.6101757325705, "total_per_gpu": 3823.826345467804, "tpot_error_pct": 5.88, "tpot_ms": 17.910296599503226, "tpot_relative": 2.565823, "ttft_error_pct": 38.86, "ttft_ms": 368.01983378750026, "ttft_relative": 1.287191, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 16265.456606118096, "interactivity_tok_s": 59.114710666454926, "output_per_gpu": 443.3758441271751, "total_per_gpu": 3965.9817519191793, "tpot_ms": 16.916263121752152, "tpot_relative": 2.423418, "ttft_ms": 601.9223929692089, "ttft_relative": 2.105292, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29610.846802118016, "interactivity_tok_s": 35.33692691538138, "output_per_gpu": 556.9429120366057, "total_per_gpu": null, "tpot_error_pct": 13.97, "tpot_ms": 28.299008637469324, "tpot_relative": 4.054107, "ttft_error_pct": 26.37, "ttft_ms": 660.9609659868967, "ttft_relative": 2.311787, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 26807.676453757507, "interactivity_tok_s": 35.171593328022105, "output_per_gpu": 542.0639217238136, "total_per_gpu": 4886.04383917442, "tpot_error_pct": 14.5, "tpot_ms": 28.43203578165094, "tpot_relative": 4.073164, "ttft_error_pct": 34.04, "ttft_ms": 592.095734567188, "ttft_relative": 2.070923, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 23798.32160983715, "interactivity_tok_s": 40.27210292590273, "output_per_gpu": 607.8553981925364, "total_per_gpu": 5480.649068420501, "tpot_ms": 24.831084729792124, "tpot_relative": 3.557293, "ttft_ms": 897.6363394020154, "ttft_relative": 3.139586, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 44394.70361696593, "interactivity_tok_s": 23.400716742360334, "output_per_gpu": 741.6450634560897, "total_per_gpu": null, "tpot_error_pct": 13.85, "tpot_ms": 42.73373380011839, "tpot_relative": 6.12202, "ttft_error_pct": 49.52, "ttft_ms": 678.0939394448243, "ttft_relative": 2.371711, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 40795.73890341746, "interactivity_tok_s": 23.15860483315622, "output_per_gpu": 712.2922887222005, "total_per_gpu": 6430.63308026559, "tpot_error_pct": 15.04, "tpot_ms": 43.18049412753474, "tpot_relative": 6.186023, "ttft_error_pct": 22.86, "ttft_ms": 1036.2284978164078, "ttft_relative": 3.624328, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 35906.07757813577, "interactivity_tok_s": 26.642034146099945, "output_per_gpu": 808.3349231344375, "total_per_gpu": 7300.281145495256, "tpot_ms": 37.53467150879646, "tpot_relative": 5.377204, "ttft_ms": 1343.3636750400183, "ttft_relative": 4.698568, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 72021.76216265939, "interactivity_tok_s": 14.343487262361258, "output_per_gpu": 913.0454021042568, "total_per_gpu": null, "tpot_error_pct": 13.96, "tpot_ms": 69.71805264010655, "tpot_relative": 9.987785, "ttft_error_pct": 70.78, "ttft_ms": 700.1943118303914, "ttft_relative": 2.44901, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 67484.61573167567, "interactivity_tok_s": 14.085366448404624, "output_per_gpu": 865.4836475902772, "total_per_gpu": 7787.113270109081, "tpot_error_pct": 16.04, "tpot_ms": 70.99566799792169, "tpot_relative": 10.170816, "ttft_error_pct": 19.24, "ttft_ms": 1935.3132442691356, "ttft_relative": 6.768979, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26252167464", "measured": {"e2e_ms": 58885.692367462885, "interactivity_tok_s": 16.345272647805654, "output_per_gpu": 991.8698436463243, "total_per_gpu": 8925.44557995703, "tpot_ms": 61.17976870420999, "tpot_relative": 8.764593, "ttft_ms": 2396.4617931235807, "ttft_relative": 8.381899, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 4}]}, {"aic": {"coverage_pct": 72.73, "points": 8, "status_counts": {"failed": 3, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.27, "tpot_shape_error_pct": 3.27, "ttft_mape_pct": 50.62, "ttft_shape_error_pct": 52.55}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb200", "precisions": ["fp8"], "rows": 11, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "8462b41b07d64a3b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 27430.71288118017, "interactivity_tok_s": 60.983520076533104, "output_per_gpu": 2102.837542262244, "total_per_gpu": 9455.544788888981, "tpot_ms": 16.3978727161866, "tpot_relative": 1.0, "ttft_ms": 12320.8171181961, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.27, "tpot_shape_error_pct": 3.27, "ttft_mape_pct": 50.62, "ttft_shape_error_pct": 52.55}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "5cee1768d8cd6621", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7411.484, "interactivity_tok_s": 144.3001443001443, "output_per_gpu": 16.604125, "total_per_gpu": null, "tpot_error_pct": 48.37, "tpot_ms": 6.93, "tpot_relative": 1.483736, "ttft_error_pct": 86.53, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 4513.768325932324, "interactivity_tok_s": 214.10328290419966, "output_per_gpu": 51.516246244412386, "total_per_gpu": 230.62975637829592, "tpot_ms": 4.6706430019919365, "tpot_relative": 1.0, "ttft_ms": 172.6787330582738, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 7589.486, "interactivity_tok_s": 140.76576576576576, "output_per_gpu": 32.384, "total_per_gpu": null, "tpot_error_pct": 40.66, "tpot_ms": 7.104, "tpot_relative": 1.52099, "ttft_error_pct": 68.09, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 4903.509530890733, "interactivity_tok_s": 198.00018725436308, "output_per_gpu": 95.01708954106131, "total_per_gpu": 423.4724491336972, "tpot_ms": 5.050500274099939, "tpot_relative": 1.081329, "ttft_ms": 191.62246994674206, "ttft_relative": 1.109705, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 8903.018, "interactivity_tok_s": 119.21793037672866, "output_per_gpu": 54.876125, "total_per_gpu": null, "tpot_error_pct": 47.01, "tpot_ms": 8.388, "tpot_relative": 1.795898, "ttft_error_pct": 43.14, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 5481.329117063433, "interactivity_tok_s": 175.26113260108423, "output_per_gpu": 163.36353389693235, "total_per_gpu": 734.4479159106278, "tpot_ms": 5.705771640059649, "tpot_relative": 1.221624, "ttft_ms": 225.01963772810996, "ttft_relative": 1.303111, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 10462.07, "interactivity_tok_s": 100.88781275221952, "output_per_gpu": 92.912625, "total_per_gpu": null, "tpot_error_pct": 54.71, "tpot_ms": 9.912, "tpot_relative": 2.122192, "ttft_error_pct": 7.72, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 6224.915298190899, "interactivity_tok_s": 156.08138512314628, "output_per_gpu": 291.8030878447253, "total_per_gpu": 1298.1768581121123, "tpot_ms": 6.406913926417378, "tpot_relative": 1.371741, "ttft_ms": 299.01380450464785, "ttft_relative": 1.731619, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 13121.87, "interactivity_tok_s": 79.923273657289, "output_per_gpu": 147.2, "total_per_gpu": null, "tpot_error_pct": 55.52, "tpot_ms": 12.512, "tpot_relative": 2.67886, "ttft_error_pct": 21.47, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 7765.040869556833, "interactivity_tok_s": 124.29402750727989, "output_per_gpu": 456.21780949589595, "total_per_gpu": 2059.1242805234942, "tpot_ms": 8.045438868262838, "tpot_relative": 1.722555, "ttft_ms": 410.13961761491373, "ttft_relative": 2.37516, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 15666.071, "interactivity_tok_s": 66.67111140742716, "output_per_gpu": 245.647375, "total_per_gpu": null, "tpot_error_pct": 43.41, "tpot_ms": 14.999, "tpot_relative": 3.211335, "ttft_error_pct": 40.55, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 10209.286200260976, "interactivity_tok_s": 95.61508050197857, "output_per_gpu": 702.9593657602001, "total_per_gpu": 3143.8028270076907, "tpot_ms": 10.458601245222054, "tpot_relative": 2.239221, "ttft_ms": 541.7819766968023, "ttft_relative": 3.137514, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 20318.675, "interactivity_tok_s": 51.158745587558194, "output_per_gpu": 376.94975, "total_per_gpu": null, "tpot_error_pct": 45.02, "tpot_ms": 19.547, "tpot_relative": 4.185077, "ttft_error_pct": 58.99, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 13193.401490902761, "interactivity_tok_s": 74.1889765639553, "output_per_gpu": 1080.9924258783924, "total_per_gpu": 4873.085607315079, "tpot_ms": 13.479091454212753, "tpot_relative": 2.885918, "ttft_ms": 785.3177040437004, "ttft_relative": 4.547854, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 27085.82, "interactivity_tok_s": 38.223377417628626, "output_per_gpu": 563.246125, "total_per_gpu": null, "tpot_error_pct": 51.49, "tpot_ms": 26.162, "tpot_relative": 5.60137, "ttft_error_pct": 78.45, "ttft_ms": 322.094, "ttft_relative": 1.865279, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 17365.298681912827, "interactivity_tok_s": 57.90477366603934, "output_per_gpu": 1640.8287291605736, "total_per_gpu": 7409.028835342278, "tpot_ms": 17.269733334377776, "tpot_relative": 3.697507, "ttft_ms": 1494.543466414325, "ttft_relative": 8.655052, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "e3f090105f29722b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 28223.422374527036, "interactivity_tok_s": 45.39879514020965, "output_per_gpu": 4021.045003284578, "total_per_gpu": 12053.925663952206, "tpot_ms": 22.027016287802347, "tpot_relative": 1.0, "ttft_ms": 7927.022208068956, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319956855", "measured": {"e2e_ms": 54383.14474440067, "interactivity_tok_s": 45.06392829691527, "output_per_gpu": 4138.284186909198, "total_per_gpu": 12419.236420369569, "tpot_ms": 22.190697477841766, "tpot_relative": 1.007431, "ttft_ms": 33959.06219082317, "ttft_relative": 4.283962, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}]}, {"aic": {"coverage_pct": 72.73, "points": 8, "status_counts": {"failed": 3, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.9, "tpot_shape_error_pct": 3.09, "ttft_mape_pct": 48.43, "ttft_shape_error_pct": 48.39}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 11, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 11, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 2, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "35f3d3f3352f27b8", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2048, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 27458.53633311944, "interactivity_tok_s": 47.933559426827934, "output_per_gpu": 4123.052519999048, "total_per_gpu": 12359.714587636523, "tpot_ms": 20.862210358622146, "tpot_relative": 1.0, "ttft_ms": 8234.927415161588, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4096, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 52918.63594184357, "interactivity_tok_s": 46.8084119952583, "output_per_gpu": 4216.059097279451, "total_per_gpu": 12652.64354173564, "tpot_ms": 21.363681384903643, "tpot_relative": 1.024037, "ttft_ms": 33255.69217776168, "ttft_relative": 4.038371, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 2, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 100.0, "points": 8, "status_counts": {"failed": 0, "success": 8, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 48.9, "tpot_shape_error_pct": 3.09, "ttft_mape_pct": 48.43, "ttft_shape_error_pct": 48.39}, "aisimulate": {"points": 0, "status_counts": {"failed": 8, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "2549e82a335db528", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": 7205.093, "interactivity_tok_s": 148.3679525222552, "output_per_gpu": 17.07525, "total_per_gpu": null, "tpot_error_pct": 50.04, "tpot_ms": 6.74, "tpot_relative": 1.500402, "ttft_error_pct": 74.0, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 4352.838803699706, "interactivity_tok_s": 222.61150532819659, "output_per_gpu": 53.420760028107416, "total_per_gpu": 239.15595116098226, "tpot_ms": 4.492130802160014, "tpot_relative": 1.0, "ttft_ms": 178.20690668886527, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 7350.359, "interactivity_tok_s": 145.30659691950015, "output_per_gpu": 33.443875, "total_per_gpu": null, "tpot_error_pct": 45.09, "tpot_ms": 6.882, "tpot_relative": 1.532012, "ttft_error_pct": 63.59, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 4620.653702155687, "interactivity_tok_s": 210.82105076783233, "output_per_gpu": 100.84480075243519, "total_per_gpu": 449.44540990785265, "tpot_ms": 4.74335933891751, "tpot_relative": 1.055926, "ttft_ms": 189.54519043909386, "ttft_relative": 1.063624, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 8831.663, "interactivity_tok_s": 120.04801920768307, "output_per_gpu": 55.2295, "total_per_gpu": null, "tpot_error_pct": 55.47, "tpot_ms": 8.33, "tpot_relative": 1.854354, "ttft_error_pct": 45.05, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 5138.269607027178, "interactivity_tok_s": 186.643027827358, "output_per_gpu": 174.03023997441113, "total_per_gpu": 782.4031716605317, "tpot_ms": 5.357821353632266, "tpot_relative": 1.192713, "ttft_ms": 213.77508195873816, "ttft_relative": 1.199589, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 10325.243, "interactivity_tok_s": 102.1450459652707, "output_per_gpu": 94.09025, "total_per_gpu": null, "tpot_error_pct": 52.85, "tpot_ms": 9.79, "tpot_relative": 2.179367, "ttft_error_pct": 12.35, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 6200.226620025933, "interactivity_tok_s": 156.12764266805365, "output_per_gpu": 292.9598551641167, "total_per_gpu": 1303.323097568815, "tpot_ms": 6.405015684033105, "tpot_relative": 1.42583, "ttft_ms": 275.9769244657946, "ttft_relative": 1.548632, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 12957.422, "interactivity_tok_s": 80.8865162177465, "output_per_gpu": 148.966375, "total_per_gpu": null, "tpot_error_pct": 53.46, "tpot_ms": 12.363, "tpot_relative": 2.752146, "ttft_error_pct": 15.77, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 7732.816798774002, "interactivity_tok_s": 124.1284815399165, "output_per_gpu": 458.85949667173946, "total_per_gpu": 2071.0474498783633, "tpot_ms": 8.056168798604258, "tpot_relative": 1.793396, "ttft_ms": 368.1357609173574, "ttft_relative": 2.065777, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 15470.933, "interactivity_tok_s": 67.47638326585695, "output_per_gpu": 248.591375, "total_per_gpu": null, "tpot_error_pct": 41.43, "tpot_ms": 14.82, "tpot_relative": 3.299103, "ttft_error_pct": 39.86, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 10201.213415249731, "interactivity_tok_s": 95.43221731807617, "output_per_gpu": 704.1510319669179, "total_per_gpu": 3149.132244001058, "tpot_ms": 10.478641575171556, "tpot_relative": 2.332666, "ttft_ms": 515.5832340464258, "ttft_relative": 2.893172, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 20265.734, "interactivity_tok_s": 51.26364894653201, "output_per_gpu": 377.656375, "total_per_gpu": null, "tpot_error_pct": 44.91, "tpot_ms": 19.507, "tpot_relative": 4.342483, "ttft_error_pct": 59.06, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 13149.616091448115, "interactivity_tok_s": 74.28532757628233, "output_per_gpu": 1085.114342950187, "total_per_gpu": 4891.6671017605995, "tpot_ms": 13.46160853868642, "tpot_relative": 2.996709, "ttft_ms": 757.3556430861572, "ttft_relative": 4.249867, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": 26570.483, "interactivity_tok_s": 38.955979742890534, "output_per_gpu": 573.96225, "total_per_gpu": null, "tpot_error_pct": 47.92, "tpot_ms": 25.67, "tpot_relative": 5.714437, "ttft_error_pct": 77.75, "ttft_ms": 310.073, "ttft_relative": 1.739961, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 1, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 17342.026473387432, "interactivity_tok_s": 57.621969136242214, "output_per_gpu": 1641.411516137058, "total_per_gpu": 7411.660362592441, "tpot_ms": 17.354491958363756, "tpot_relative": 3.86331, "ttft_ms": 1393.5763764640797, "ttft_relative": 7.819991, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 8, "serving": "disaggregated", "spec_method": "none", "total_gpus": 8}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "c54c16a80346acae", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "RuntimeError: AISimulate replay failed: engine error: generalized engine worker 0 is now poisoned because a rank may have been partially mutated: executing attention-DP rank 0: external prefill prediction failed: AIC static_ctx evidence prediction failed: PerfDataNotAvailableError: perf database error: context attention data missing for ContextKey { fmha_quant: \"fp8\", kv_quant: \"bfloat16\", n_kv_lookup: 2, head_size: 256, window_size: 0 }: tried lane_order [\"triton\", \"trtllm_mha\", \"flashinfer\", \"fa3\", \"fla\", \"default\"], then fell back through table lanes [] at [path]", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1024, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "35319969159", "measured": {"e2e_ms": 26408.60113420243, "interactivity_tok_s": 63.637685434171146, "output_per_gpu": 2160.9093657309572, "total_per_gpu": 9716.668492810562, "tpot_ms": 15.71395931793327, "tpot_relative": 1.0, "ttft_ms": 11928.676738861735, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}]}, {"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.62, "tpot_shape_error_pct": 6.45, "ttft_mape_pct": 28.76, "ttft_shape_error_pct": 26.4}, "aisimulate": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.13, "tpot_shape_error_pct": 9.06, "ttft_mape_pct": 63.68, "ttft_shape_error_pct": 25.84}, "gpu": "h100", "precisions": ["fp8"], "rows": 4, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 15.62, "tpot_shape_error_pct": 6.45, "ttft_mape_pct": 28.76, "ttft_shape_error_pct": 26.4}, "aisimulate": {"points": 4, "status_counts": {"failed": 0, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.13, "tpot_shape_error_pct": 9.06, "ttft_mape_pct": 63.68, "ttft_shape_error_pct": 25.84}, "framework": "sglang", "id": "21c1c4019d32fda6", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 5741.173833948304, "interactivity_tok_s": 190.12641758706164, "output_per_gpu": 22.956201746204414, "total_per_gpu": null, "tpot_error_pct": 19.51, "tpot_ms": 5.259658350960542, "tpot_relative": 0.804881, "ttft_error_pct": 36.61, "ttft_ms": 360.5433409156681, "ttft_relative": 0.633872, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5057.2135826999975, "interactivity_tok_s": 190.31854002973438, "output_per_gpu": 22.994381806970317, "total_per_gpu": 205.83469196573796, "tpot_error_pct": 19.59, "tpot_ms": 5.254348839812271, "tpot_relative": 0.804069, "ttft_error_pct": 69.35, "ttft_ms": 174.32982259999972, "ttft_relative": 0.30649, "unavailable_metrics": {}}, "concurrency": 1, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34776525730", "measured": {"e2e_ms": 6643.379681743681, "interactivity_tok_s": 153.0291672546896, "output_per_gpu": 17.5013279443133, "total_per_gpu": 156.66350487913945, "tpot_ms": 6.534701965251364, "tpot_relative": 1.0, "ttft_ms": 568.7953190878034, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 6313.867474928996, "interactivity_tok_s": 172.1534808390966, "output_per_gpu": 41.70143531401569, "total_per_gpu": null, "tpot_error_pct": 13.3, "tpot_ms": 5.808770145836615, "tpot_relative": 0.888911, "ttft_error_pct": 20.72, "ttft_ms": 371.4956157381384, "ttft_relative": 0.653127, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 5598.4531392000135, "interactivity_tok_s": 172.57210256228407, "output_per_gpu": 41.64438228905133, "total_per_gpu": 371.112560244837, "tpot_error_pct": 13.51, "tpot_ms": 5.794679355193484, "tpot_relative": 0.886755, "ttft_error_pct": 60.87, "ttft_ms": 183.3452493500002, "ttft_relative": 0.32234, "unavailable_metrics": {}}, "concurrency": 2, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34776525730", "measured": {"e2e_ms": 6730.464653833769, "interactivity_tok_s": 149.26417563438187, "output_per_gpu": 34.6303629119366, "total_per_gpu": 308.60735436662924, "tpot_ms": 6.699531188578497, "tpot_relative": 1.025224, "ttft_ms": 468.596677982714, "ttft_relative": 0.823841, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 7412.579591266253, "interactivity_tok_s": 145.68712984750778, "output_per_gpu": 67.4811045705857, "total_per_gpu": null, "tpot_error_pct": 16.65, "tpot_ms": 6.864024303634167, "tpot_relative": 1.050396, "ttft_error_pct": 11.69, "ttft_ms": 390.6827286485008, "ttft_relative": 0.68686, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 6798.5280132749995, "interactivity_tok_s": 139.31484177253506, "output_per_gpu": 66.22054802876656, "total_per_gpu": 595.2832337469789, "tpot_error_pct": 12.84, "tpot_ms": 7.177986116028758, "tpot_relative": 1.098441, "ttft_error_pct": 54.61, "ttft_ms": 200.79426442500016, "ttft_relative": 0.353017, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34776525730", "measured": {"e2e_ms": 8010.5121047236025, "interactivity_tok_s": 121.43252621551761, "output_per_gpu": 56.265187637881176, "total_per_gpu": 505.7904810740708, "tpot_ms": 8.235025912457811, "tpot_relative": 1.260199, "ttft_ms": 442.4172525992617, "ttft_relative": 0.777815, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9857.701626074195, "interactivity_tok_s": 108.50063523311482, "output_per_gpu": 102.48108044374366, "total_per_gpu": null, "tpot_error_pct": 13.03, "tpot_ms": 9.216535901854296, "tpot_relative": 1.410399, "ttft_error_pct": 46.0, "ttft_ms": 429.18539847725043, "ttft_relative": 0.754552, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9016.428945600008, "interactivity_tok_s": 105.52698786447058, "output_per_gpu": 101.25468989695236, "total_per_gpu": 900.7072806427003, "tpot_error_pct": 10.57, "tpot_ms": 9.476248874688913, "tpot_relative": 1.450142, "ttft_error_pct": 69.9, "ttft_ms": 239.24836217499995, "ttft_relative": 0.420623, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "34776525730", "measured": {"e2e_ms": 10609.469724539667, "interactivity_tok_s": 94.3679739754566, "output_per_gpu": 86.3080770386625, "total_per_gpu": 767.7502488636254, "tpot_ms": 10.596815401166522, "tpot_relative": 1.621622, "ttft_ms": 794.8539138829801, "ttft_relative": 1.397434, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 4, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.08, "tpot_shape_error_pct": 3.67, "ttft_mape_pct": 20.77, "ttft_shape_error_pct": 17.41}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.17, "tpot_shape_error_pct": 4.57, "ttft_mape_pct": 49.39, "ttft_shape_error_pct": 16.23}, "gpu": "h200", "precisions": ["fp8"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.08, "tpot_shape_error_pct": 3.67, "ttft_mape_pct": 20.77, "ttft_shape_error_pct": 17.41}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.17, "tpot_shape_error_pct": 4.57, "ttft_mape_pct": 49.39, "ttft_shape_error_pct": 16.23}, "framework": "sglang", "id": "ec80a0b3489fa718", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 8, "moe_tp_size": 1, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 7891.750470124665, "interactivity_tok_s": 135.96687716304538, "output_per_gpu": 63.560014355576676, "total_per_gpu": null, "tpot_error_pct": 14.52, "tpot_ms": 7.354732423550809, "tpot_relative": 0.854772, "ttft_error_pct": 7.95, "ttft_ms": 367.8592008321876, "ttft_relative": 0.920514, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 7217.533620249975, "interactivity_tok_s": 130.75173293542733, "output_per_gpu": 62.3168962793741, "total_per_gpu": 560.1917326046005, "tpot_error_pct": 11.11, "tpot_ms": 7.648082190190605, "tpot_relative": 0.888865, "ttft_error_pct": 53.03, "ttft_ms": 187.71483377500005, "ttft_relative": 0.469729, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 8306.738974526525, "interactivity_tok_s": 116.2206908622079, "output_per_gpu": 54.163487991846715, "total_per_gpu": 486.8974546812239, "tpot_ms": 8.604319872660259, "tpot_relative": 1.0, "ttft_ms": 399.6235067024827, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 9488.24016916962, "interactivity_tok_s": 112.61364732719957, "output_per_gpu": 106.50508845474843, "total_per_gpu": null, "tpot_error_pct": 18.2, "tpot_ms": 8.879918408951754, "tpot_relative": 1.03203, "ttft_error_pct": 26.21, "ttft_ms": 404.08363681197375, "ttft_relative": 1.011161, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 8682.511657074972, "interactivity_tok_s": 109.48196583551474, "output_per_gpu": 104.99294084795216, "total_per_gpu": 933.9607511916863, "tpot_error_pct": 15.86, "tpot_ms": 9.133924408175096, "tpot_relative": 1.061551, "ttft_error_pct": 59.19, "ttft_ms": 223.50695465000013, "ttft_relative": 0.559294, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 10604.043644899502, "interactivity_tok_s": 92.11781818523797, "output_per_gpu": 86.02012482096495, "total_per_gpu": 765.1887807556163, "tpot_ms": 10.855663102974487, "tpot_relative": 1.261653, "ttft_ms": 547.6442037150264, "ttft_relative": 1.3704, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13995.055052066942, "interactivity_tok_s": 75.67545468942836, "output_per_gpu": 145.81373306249324, "total_per_gpu": null, "tpot_error_pct": 12.16, "tpot_ms": 13.214324302430615, "tpot_relative": 1.535778, "ttft_error_pct": 16.47, "ttft_ms": 476.80129068042424, "ttft_relative": 1.193126, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12590.96100366877, "interactivity_tok_s": 74.38756878468142, "output_per_gpu": 141.87379633961, "total_per_gpu": 1280.3755013682567, "tpot_error_pct": 10.64, "tpot_ms": 13.443106372982166, "tpot_relative": 1.562367, "ttft_error_pct": 49.19, "ttft_ms": 290.02743003125045, "ttft_relative": 0.725752, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 14348.193249676842, "interactivity_tok_s": 66.47612149675004, "output_per_gpu": 124.55551824052006, "total_per_gpu": 1124.082376238381, "tpot_ms": 15.042995552153101, "tpot_relative": 1.748307, "ttft_ms": 570.7883091643453, "ttft_relative": 1.428315, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19354.074490164665, "interactivity_tok_s": 54.615031810822856, "output_per_gpu": 212.7239500410143, "total_per_gpu": null, "tpot_error_pct": 17.55, "tpot_ms": 18.309977433755403, "tpot_relative": 2.127998, "ttft_error_pct": 17.62, "ttft_ms": 622.9675754328886, "ttft_relative": 1.558886, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 17567.09485719386, "interactivity_tok_s": 53.97233499603669, "output_per_gpu": 206.4279744856754, "total_per_gpu": 1845.944675960053, "tpot_error_pct": 16.57, "tpot_ms": 18.528010694246085, "tpot_relative": 2.153338, "ttft_error_pct": 44.98, "ttft_ms": 416.0439879906251, "ttft_relative": 1.04109, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 21329.864761338104, "interactivity_tok_s": 45.02755111527298, "output_per_gpu": 170.25327724073188, "total_per_gpu": 1522.4590149194557, "tpot_ms": 22.208625058021603, "tpot_relative": 2.581102, "ttft_ms": 756.1991440772545, "ttft_relative": 1.892279, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 29658.027360116444, "interactivity_tok_s": 35.360504971921124, "output_per_gpu": 278.1818372922615, "total_per_gpu": null, "tpot_error_pct": 17.98, "tpot_ms": 28.28013912114871, "tpot_relative": 3.286737, "ttft_error_pct": 35.59, "ttft_ms": 727.4450391813118, "ttft_relative": 1.820326, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27162.65467147211, "interactivity_tok_s": 34.811412843190205, "output_per_gpu": 267.57392849916425, "total_per_gpu": 2411.8519836359064, "tpot_error_pct": 16.69, "tpot_ms": 28.726211271704237, "tpot_relative": 3.33858, "ttft_error_pct": 40.58, "ttft_ms": 671.0364724156236, "ttft_relative": 1.679172, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28719990565", "measured": {"e2e_ms": 32945.76599446591, "interactivity_tok_s": 29.001308548019786, "output_per_gpu": 221.00161959702712, "total_per_gpu": 1992.0595313661238, "tpot_ms": 34.481202747945666, "tpot_relative": 4.007429, "ttft_ms": 1129.3540400765778, "ttft_relative": 2.826045, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 55}]}, {"aic": {"coverage_pct": 47.37, "points": 36, "status_counts": {"failed": 40, "success": 36, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.66, "tpot_shape_error_pct": 12.57, "ttft_mape_pct": 31.43, "ttft_shape_error_pct": 32.93}, "aisimulate": {"coverage_pct": 75.0, "points": 57, "status_counts": {"failed": 19, "success": 57, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.87, "tpot_shape_error_pct": 12.22, "ttft_mape_pct": 45.3, "ttft_shape_error_pct": 27.53}, "frameworks": ["dynamo-sglang", "sglang"], "gpu_skus": ["b200", "b300", "gb300", "h200"], "hf_model_paths": ["zai-org/GLM-5-FP8"], "model": "zai-org/GLM-5-FP8", "precisions": ["fp8"], "rows": 76, "workloads": [{"aic": {"coverage_pct": 48.72, "points": 19, "status_counts": {"failed": 20, "success": 19, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.84, "tpot_shape_error_pct": 10.35, "ttft_mape_pct": 31.08, "ttft_shape_error_pct": 27.96}, "aisimulate": {"coverage_pct": 74.36, "points": 29, "status_counts": {"failed": 10, "success": 29, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.75, "tpot_shape_error_pct": 14.07, "ttft_mape_pct": 51.12, "ttft_shape_error_pct": 25.42}, "gpu_skus": ["b200", "b300", "gb300", "h200"], "gpus": [{"aic": {"coverage_pct": 41.18, "points": 7, "status_counts": {"failed": 10, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.6, "tpot_shape_error_pct": 11.85, "ttft_mape_pct": 35.7, "ttft_shape_error_pct": 25.7}, "aisimulate": {"coverage_pct": 100.0, "points": 17, "status_counts": {"failed": 0, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.56, "tpot_shape_error_pct": 17.65, "ttft_mape_pct": 49.38, "ttft_shape_error_pct": 24.6}, "gpu": "b200", "precisions": ["fp8"], "rows": 17, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 6, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 6, "status_counts": {"failed": 0, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 25.32, "tpot_shape_error_pct": 24.26, "ttft_mape_pct": 65.54, "ttft_shape_error_pct": 43.82}, "framework": "dynamo-sglang", "id": "28cfdd76bda042ec", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15826.04498828754, "interactivity_tok_s": 113.0575299665958, "output_per_gpu": 12.366373335331607, "total_per_gpu": 24.864390914353578, "tpot_error_pct": 37.88, "tpot_ms": 8.84505437448936, "tpot_relative": 0.621202, "ttft_error_pct": 45.73, "ttft_ms": 7732.989765418798, "ttft_relative": 0.542744, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 27276.78848287451, "interactivity_tok_s": 70.23158220226698, "output_per_gpu": 7.5934239039472935, "total_per_gpu": 13.567850344933735, "tpot_ms": 14.238608452818275, "tpot_relative": 1.0, "ttft_ms": 14247.948317668579, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 9808.745802209363, "interactivity_tok_s": 96.49172091533632, "output_per_gpu": 40.54150137636628, "total_per_gpu": 80.95475938192422, "tpot_error_pct": 35.91, "tpot_ms": 10.363583429892593, "tpot_relative": 0.727851, "ttft_error_pct": 59.2, "ttft_ms": 245.6940090281265, "ttft_relative": 0.017244, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 15530.116732643637, "interactivity_tok_s": 61.83769330452383, "output_per_gpu": 28.272301427522184, "total_per_gpu": 50.183668396670505, "tpot_ms": 16.171366468594382, "tpot_relative": 1.135741, "ttft_ms": 602.2572443718673, "ttft_relative": 0.04227, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 12258.407851404698, "interactivity_tok_s": 76.97453643471091, "output_per_gpu": 64.40700229810446, "total_per_gpu": 128.84304424901867, "tpot_error_pct": 25.56, "tpot_ms": 12.99130915647918, "tpot_relative": 0.9124, "ttft_error_pct": 75.34, "ttft_ms": 296.37775059687453, "ttft_relative": 0.020801, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 17267.2768903386, "interactivity_tok_s": 57.29766558684626, "output_per_gpu": 51.287223533378324, "total_per_gpu": 91.24337526294512, "tpot_ms": 17.452718007931697, "tpot_relative": 1.225732, "ttft_ms": 1201.8232110513054, "ttft_relative": 0.084351, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14136.788193998442, "interactivity_tok_s": 66.76260523095196, "output_per_gpu": 111.69545819554565, "total_per_gpu": 223.63629334844094, "tpot_error_pct": 23.37, "tpot_ms": 14.97844484259862, "tpot_relative": 1.05196, "ttft_error_pct": 74.48, "ttft_ms": 373.09572453593876, "ttft_relative": 0.026186, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 19425.563092625565, "interactivity_tok_s": 51.15895627442908, "output_per_gpu": 90.92037318937929, "total_per_gpu": 161.85512947618014, "tpot_ms": 19.546919499994427, "tpot_relative": 1.372811, "ttft_ms": 1461.721785734403, "ttft_relative": 0.102592, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 16806.559660271036, "interactivity_tok_s": 56.5062186496118, "output_per_gpu": 188.40855598815222, "total_per_gpu": 376.63501571805404, "tpot_error_pct": 18.4, "tpot_ms": 17.697167212707658, "tpot_relative": 1.2429, "ttft_error_pct": 72.9, "ttft_ms": 509.5776835882818, "ttft_relative": 0.035765, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 21851.319348635752, "interactivity_tok_s": 46.1101344127604, "output_per_gpu": 163.16163116955957, "total_per_gpu": 290.02026120587044, "tpot_ms": 21.68720635356167, "tpot_relative": 1.523127, "ttft_ms": 1880.543330675755, "ttft_relative": 0.131987, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 21961.982874151443, "interactivity_tok_s": 43.644615518627624, "output_per_gpu": 288.8195137368218, "total_per_gpu": 577.37297298839, "tpot_error_pct": 10.8, "tpot_ms": 22.912333815226248, "tpot_relative": 1.609169, "ttft_error_pct": 65.62, "ttft_ms": 855.2617636445326, "ttft_relative": 0.060027, "unavailable_metrics": {}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 26150.769290334694, "interactivity_tok_s": 38.932083763561145, "output_per_gpu": 271.19028784456077, "total_per_gpu": 482.05527075013316, "tpot_ms": 25.685755894113214, "tpot_relative": 1.803951, "ttft_ms": 2487.604631097901, "ttft_relative": 0.174594, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 6, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 0.18, "tpot_shape_error_pct": null, "ttft_mape_pct": 49.72, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "f38be4795085abf8", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 44959.04922852814, "interactivity_tok_s": 21.22113691077461, "output_per_gpu": 1028.4333063570236, "total_per_gpu": 2056.4199389750374, "tpot_error_pct": 0.18, "tpot_ms": 47.12282872517871, "tpot_relative": 1.00182, "ttft_error_pct": 49.72, "ttft_ms": 1557.5608201359682, "ttft_relative": 0.502824, "unavailable_metrics": {}}, "concurrency": 1248, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 46422.420073808666, "interactivity_tok_s": 21.259762998194898, "output_per_gpu": 1482.8544431868397, "total_per_gpu": 1977.4187176136104, "tpot_ms": 47.037212977628535, "tpot_relative": 1.0, "ttft_ms": 3097.62459148106, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.03, "tpot_shape_error_pct": null, "ttft_mape_pct": 59.32, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "ae406cc514cbc31d", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 28020.4634053519, "interactivity_tok_s": 33.905425706148385, "output_per_gpu": 453.7678919094645, "total_per_gpu": 907.1515184154807, "tpot_error_pct": 17.03, "tpot_ms": 29.493804580623827, "tpot_relative": 0.829659, "ttft_error_pct": 59.32, "ttft_ms": 852.1071194810813, "ttft_relative": 0.406781, "unavailable_metrics": {}}, "concurrency": 576, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 34843.71718284795, "interactivity_tok_s": 28.129928152751198, "output_per_gpu": 455.5891930023278, "total_per_gpu": 728.8305934128792, "tpot_ms": 35.54932648849289, "tpot_relative": 1.0, "ttft_ms": 2094.757545602753, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.72, "tpot_shape_error_pct": null, "ttft_mape_pct": 53.96, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "2f1d97ca35b6be2b", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 33079.560416978704, "interactivity_tok_s": 28.768113954729703, "output_per_gpu": 668.7092967088365, "total_per_gpu": 1337.4641287556594, "tpot_error_pct": 11.72, "tpot_ms": 34.76070769093961, "tpot_relative": 0.88277, "ttft_error_pct": 53.96, "ttft_ms": 1081.2937345702499, "ttft_relative": 0.460369, "unavailable_metrics": {}}, "concurrency": 800, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 38597.798346216325, "interactivity_tok_s": 25.39563064686723, "output_per_gpu": 760.8103267234998, "total_per_gpu": 1141.5951996561769, "tpot_ms": 39.3768524162781, "tpot_relative": 1.0, "ttft_ms": 2348.756243168842, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 4.23, "tpot_shape_error_pct": null, "ttft_mape_pct": 10.18, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "cbfb16b75793d28b", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 85267.59716217463, "interactivity_tok_s": 11.46651470290099, "output_per_gpu": 1666.7137906159685, "total_per_gpu": 3332.935895547241, "tpot_error_pct": 4.23, "tpot_ms": 87.21045809560627, "tpot_relative": 1.042329, "ttft_error_pct": 10.18, "ttft_ms": 4966.828913968691, "ttft_relative": 0.898208, "unavailable_metrics": {}}, "concurrency": 2576, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 82605.84046626117, "interactivity_tok_s": 11.951875423135736, "output_per_gpu": 3448.4949715545054, "total_per_gpu": 3449.040122983348, "tpot_ms": 83.66887744364027, "tpot_relative": 1.0, "ttft_ms": 5529.709042299889, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 9.6, "tpot_shape_error_pct": 11.85, "ttft_mape_pct": 35.7, "ttft_shape_error_pct": 25.7}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.92, "tpot_shape_error_pct": 12.15, "ttft_mape_pct": 39.01, "ttft_shape_error_pct": 8.59}, "framework": "sglang", "id": "54a9c1184106945d", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10960.81749905786, "interactivity_tok_s": 94.81885244429552, "output_per_gpu": 46.41834456850437, "total_per_gpu": null, "tpot_error_pct": 17.94, "tpot_ms": 10.546425887061682, "tpot_relative": 0.820601, "ttft_error_pct": 33.61, "ttft_ms": 171.82381659375903, "ttft_relative": 1.336087, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9801.70457559999, "interactivity_tok_s": 94.32377973385073, "output_per_gpu": 45.41205012618162, "total_per_gpu": 91.26821275870498, "tpot_error_pct": 17.51, "tpot_ms": 10.601780408097047, "tpot_relative": 0.824908, "ttft_error_pct": 36.3, "ttft_ms": 81.91683677500002, "ttft_relative": 0.636978, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 11911.667631726232, "interactivity_tok_s": 77.80845923543801, "output_per_gpu": 37.31614108464534, "total_per_gpu": 74.99721977721755, "tpot_ms": 12.852073024272766, "tpot_relative": 1.0, "ttft_ms": 128.60227677556395, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 14147.007324585933, "interactivity_tok_s": 73.29327428534977, "output_per_gpu": 72.12938122560661, "total_per_gpu": null, "tpot_error_pct": 8.11, "tpot_ms": 13.643816704200443, "tpot_relative": 1.061604, "ttft_error_pct": 43.5, "ttft_ms": 189.38283618887993, "ttft_relative": 1.472624, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12768.693540100012, "interactivity_tok_s": 73.21853249148961, "output_per_gpu": 71.41199405518259, "total_per_gpu": 142.2965120624641, "tpot_error_pct": 8.02, "tpot_ms": 13.657744371156753, "tpot_relative": 1.062688, "ttft_error_pct": 33.56, "ttft_ms": 87.68848823750002, "ttft_relative": 0.681858, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 13916.383619151202, "interactivity_tok_s": 67.34619501626916, "output_per_gpu": 65.29859092834018, "total_per_gpu": 130.1148617207971, "tpot_ms": 14.848648832475613, "tpot_relative": 1.15535, "ttft_ms": 131.9783291510248, "ttft_relative": 1.026252, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16727.985622800006, "interactivity_tok_s": 61.98290328602893, "output_per_gpu": 122.33016456857762, "total_per_gpu": null, "tpot_error_pct": 7.98, "tpot_ms": 16.13348112116268, "tpot_relative": 1.255321, "ttft_error_pct": 54.12, "ttft_ms": 223.43443585058407, "ttft_relative": 1.737407, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14922.942641687481, "interactivity_tok_s": 61.736259559452634, "output_per_gpu": 120.2406402415586, "total_per_gpu": 241.76128293138717, "tpot_error_pct": 7.62, "tpot_ms": 16.19793630414214, "tpot_relative": 1.260336, "ttft_error_pct": 32.41, "ttft_ms": 97.98527567499984, "ttft_relative": 0.761925, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 16191.519680375677, "interactivity_tok_s": 57.03490816471548, "output_per_gpu": 110.58062532511745, "total_per_gpu": 222.33841895924445, "tpot_ms": 17.533121945458795, "tpot_relative": 1.364225, "ttft_ms": 144.9733052204465, "ttft_relative": 1.1273, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20705.62092912606, "interactivity_tok_s": 50.117806489703774, "output_per_gpu": 198.37687356565047, "total_per_gpu": null, "tpot_error_pct": 9.54, "tpot_ms": 19.95298817009161, "tpot_relative": 1.552511, "ttft_error_pct": 59.03, "ttft_ms": 293.71403112234185, "ttft_relative": 2.283894, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18634.463223990653, "interactivity_tok_s": 49.85320518271241, "output_per_gpu": 193.79384210961464, "total_per_gpu": 386.9746636178629, "tpot_error_pct": 9.06, "tpot_ms": 20.05889082427081, "tpot_relative": 1.560751, "ttft_error_pct": 35.99, "ttft_ms": 118.22417852499989, "ttft_relative": 0.919301, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 20545.097157940338, "interactivity_tok_s": 45.33664483135459, "output_per_gpu": 175.31096685230946, "total_per_gpu": 350.06737927112385, "tpot_ms": 22.057212299671658, "tpot_relative": 1.716238, "ttft_ms": 184.69102545309397, "ttft_relative": 1.436141, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28148.9626767273, "interactivity_tok_s": 36.79466770669647, "output_per_gpu": 292.20116668433957, "total_per_gpu": null, "tpot_error_pct": 3.9, "tpot_ms": 27.17785109438573, "tpot_relative": 2.114667, "ttft_error_pct": 5.11, "ttft_ms": 346.02100717069743, "ttft_relative": 2.690629, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25561.170251390595, "interactivity_tok_s": 36.27315631268495, "output_per_gpu": 281.81222341598345, "total_per_gpu": 563.7515095554836, "tpot_error_pct": 2.52, "tpot_ms": 27.568596219742084, "tpot_relative": 2.14507, "ttft_error_pct": 52.55, "ttft_ms": 156.20542428437528, "ttft_relative": 1.21464, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 26389.81387793267, "interactivity_tok_s": 35.359944340290554, "output_per_gpu": 270.56971526434467, "total_per_gpu": 541.2614242609196, "tpot_ms": 28.28058750252498, "tpot_relative": 2.200469, "ttft_ms": 329.21150891711477, "ttft_relative": 2.55992, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35046.40459387468, "interactivity_tok_s": 29.485860769725385, "output_per_gpu": 469.1231783218735, "total_per_gpu": null, "tpot_error_pct": 12.51, "tpot_ms": 33.91456019580579, "tpot_relative": 2.63884, "ttft_error_pct": 16.4, "ttft_ms": 351.8095135653574, "ttft_relative": 2.73564, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32119.237729818717, "interactivity_tok_s": 28.846726636513985, "output_per_gpu": 447.5062326663642, "total_per_gpu": 895.9955645521651, "tpot_error_pct": 10.57, "tpot_ms": 34.66597831360897, "tpot_relative": 2.697306, "ttft_error_pct": 45.07, "ttft_ms": 231.1512865054695, "ttft_relative": 1.797412, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 36073.967055510984, "interactivity_tok_s": 25.797165966901147, "output_per_gpu": 397.60629233827757, "total_per_gpu": 796.0860617526439, "tpot_ms": 38.76394799657614, "tpot_relative": 3.016163, "ttft_ms": 420.82608436018063, "ttft_relative": 3.272307, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 52131.61762966123, "interactivity_tok_s": 19.765878429102802, "output_per_gpu": 630.3082462845257, "total_per_gpu": null, "tpot_error_pct": 7.24, "tpot_ms": 50.592236696529724, "tpot_relative": 3.936504, "ttft_error_pct": 38.11, "ttft_ms": 375.7594891113288, "ttft_relative": 2.921873, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 47089.61492522037, "interactivity_tok_s": 19.74962643002475, "output_per_gpu": 614.3349014334658, "total_per_gpu": 1228.0760501773207, "tpot_error_pct": 7.16, "tpot_ms": 50.6338691287715, "tpot_relative": 3.939743, "ttft_error_pct": 37.16, "ttft_ms": 381.5674910578132, "ttft_relative": 2.967035, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 50915.76981519974, "interactivity_tok_s": 18.335778044308288, "output_per_gpu": 566.6809357220208, "total_per_gpu": 1132.8141761577228, "tpot_ms": 54.538181994977606, "tpot_relative": 4.243532, "ttft_ms": 607.1708605245703, "ttft_relative": 4.721307, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.09, "tpot_shape_error_pct": 9.33, "ttft_mape_pct": 33.55, "ttft_shape_error_pct": 36.6}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.55, "tpot_shape_error_pct": 9.97, "ttft_mape_pct": 47.56, "ttft_shape_error_pct": 32.11}, "gpu": "b300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.09, "tpot_shape_error_pct": 9.33, "ttft_mape_pct": 33.55, "ttft_shape_error_pct": 36.6}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.55, "tpot_shape_error_pct": 9.97, "ttft_mape_pct": 47.56, "ttft_shape_error_pct": 32.11}, "framework": "sglang", "id": "c1a72e7093b672fb", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 10995.794744701938, "interactivity_tok_s": 94.5052768986334, "output_per_gpu": 46.27335768085267, "total_per_gpu": null, "tpot_error_pct": 20.45, "tpot_ms": 10.581419713447351, "tpot_relative": 0.795495, "ttft_error_pct": 53.18, "ttft_ms": 171.0023778452974, "ttft_relative": 1.531799, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 9794.034961824846, "interactivity_tok_s": 94.38724092859246, "output_per_gpu": 45.48896243197821, "total_per_gpu": 91.4227895432743, "tpot_error_pct": 20.35, "tpot_ms": 10.594652308531172, "tpot_relative": 0.79649, "ttft_error_pct": 27.65, "ttft_ms": 80.76276817500043, "ttft_relative": 0.723454, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 12307.615644676844, "interactivity_tok_s": 75.17849037612565, "output_per_gpu": 36.13752426527009, "total_per_gpu": 72.62845971611407, "tpot_ms": 13.301677048806088, "tpot_relative": 1.0, "ttft_ms": 111.63499085814692, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 13536.699433116935, "interactivity_tok_s": 76.63838183831194, "output_per_gpu": 75.36817490076002, "total_per_gpu": null, "tpot_error_pct": 15.12, "tpot_ms": 13.048292200502786, "tpot_relative": 0.980951, "ttft_error_pct": 34.64, "ttft_ms": 188.29651200258473, "ttft_relative": 1.686716, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12217.178494137477, "interactivity_tok_s": 76.53805520949963, "output_per_gpu": 74.6163740464421, "total_per_gpu": 148.68160328014713, "tpot_error_pct": 15.0, "tpot_ms": 13.065395995009336, "tpot_relative": 0.982237, "ttft_error_pct": 38.22, "ttft_ms": 86.39436383750073, "ttft_relative": 0.7739, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 14409.996830387536, "interactivity_tok_s": 65.05390875732242, "output_per_gpu": 63.09973620070541, "total_per_gpu": 125.73339383974749, "tpot_ms": 15.371866488920864, "tpot_relative": 1.155634, "ttft_ms": 139.85318670020206, "ttft_relative": 1.252772, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16177.83347596624, "interactivity_tok_s": 64.11384939930024, "output_per_gpu": 126.48415708067309, "total_per_gpu": null, "tpot_error_pct": 14.4, "tpot_ms": 15.597254093604842, "tpot_relative": 1.172578, "ttft_error_pct": 24.87, "ttft_ms": 221.84253820848573, "ttft_relative": 1.987213, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14437.14504708749, "interactivity_tok_s": 63.82064718954222, "output_per_gpu": 124.27379997325157, "total_per_gpu": 249.87053674975047, "tpot_error_pct": 14.01, "tpot_ms": 15.668910361094897, "tpot_relative": 1.177965, "ttft_error_pct": 45.76, "ttft_ms": 96.36152858124981, "ttft_relative": 0.863184, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 16852.542369320872, "interactivity_tok_s": 54.88013477947079, "output_per_gpu": 106.30197387763124, "total_per_gpu": 213.73556836661277, "tpot_ms": 18.221529593875445, "tpot_relative": 1.369867, "ttft_ms": 177.66582944459515, "ttft_relative": 1.591489, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 20381.397523131858, "interactivity_tok_s": 50.920546592402836, "output_per_gpu": 201.53537048139702, "total_per_gpu": null, "tpot_error_pct": 11.7, "tpot_ms": 19.638438055360474, "tpot_relative": 1.476388, "ttft_error_pct": 22.69, "ttft_ms": 291.2753924980933, "ttft_relative": 2.609176, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18339.982968409262, "interactivity_tok_s": 50.650961260999544, "output_per_gpu": 196.95625702673613, "total_per_gpu": 393.289489906724, "tpot_error_pct": 11.23, "tpot_ms": 19.742961932096332, "tpot_relative": 1.484246, "ttft_error_pct": 51.4, "ttft_ms": 115.39247942187453, "ttft_relative": 1.033659, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 20766.97869815689, "interactivity_tok_s": 44.96227012080255, "output_per_gpu": 173.50104362508202, "total_per_gpu": 346.4532580771473, "tpot_ms": 22.240869896320763, "tpot_relative": 1.672035, "ttft_ms": 237.4165107794397, "ttft_relative": 2.126721, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 28027.767825549243, "interactivity_tok_s": 36.952668151660866, "output_per_gpu": 293.4625160065935, "total_per_gpu": null, "tpot_error_pct": 8.54, "tpot_ms": 27.06164534305906, "tpot_relative": 2.034454, "ttft_error_pct": 11.48, "ttft_ms": 343.7046395998262, "ttft_relative": 3.078825, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 25407.075588815667, "interactivity_tok_s": 36.491253276011804, "output_per_gpu": 283.63941122020896, "total_per_gpu": 567.4067090013679, "tpot_error_pct": 7.38, "tpot_ms": 27.40382722501253, "tpot_relative": 2.060178, "ttft_error_pct": 60.29, "ttft_ms": 154.17030331250012, "ttft_relative": 1.381021, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 27648.071604380493, "interactivity_tok_s": 33.797162019733314, "output_per_gpu": 258.5270509377954, "total_per_gpu": 517.17066584431, "tpot_ms": 29.588283164607876, "tpot_relative": 2.224402, "ttft_ms": 388.2655395639631, "ttft_relative": 3.477991, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35194.505350183346, "interactivity_tok_s": 29.35824487731495, "output_per_gpu": 467.135194404371, "total_per_gpu": null, "tpot_error_pct": 14.85, "tpot_ms": 34.061981708337676, "tpot_relative": 2.560728, "ttft_error_pct": 34.57, "ttft_ms": 349.09806255389987, "ttft_relative": 3.127138, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32225.419745146966, "interactivity_tok_s": 28.748258559268873, "output_per_gpu": 446.26987820280175, "total_per_gpu": 893.5201395531303, "tpot_error_pct": 13.04, "tpot_ms": 34.78471567028483, "tpot_relative": 2.615062, "ttft_error_pct": 57.06, "ttft_ms": 229.08476676875028, "ttft_relative": 2.052087, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 37323.78873400808, "interactivity_tok_s": 24.998693884025176, "output_per_gpu": 384.5988081404646, "total_per_gpu": 770.0425180062668, "tpot_ms": 40.00208989474552, "tpot_relative": 3.007297, "ttft_ms": 533.5119621523518, "ttft_relative": 4.779075, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 51224.95926929246, "interactivity_tok_s": 20.116874888777208, "output_per_gpu": 641.4734740124574, "total_per_gpu": null, "tpot_error_pct": 13.54, "tpot_ms": 49.70951032547702, "tpot_relative": 3.737086, "ttft_error_pct": 53.43, "ttft_ms": 372.13020632946944, "ttft_relative": 3.333455, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 46092.15321442912, "interactivity_tok_s": 20.179635099229742, "output_per_gpu": 627.5470173391401, "total_per_gpu": 1254.4875125215008, "tpot_error_pct": 13.81, "tpot_ms": 49.554909941764514, "tpot_relative": 3.725463, "ttft_error_pct": 52.55, "ttft_ms": 379.2003381269522, "ttft_relative": 3.396787, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 53825.24277566463, "interactivity_tok_s": 17.39250865750861, "output_per_gpu": 536.6981568628275, "total_per_gpu": 1072.8775966978533, "tpot_ms": 57.49601852682044, "tpot_relative": 4.322464, "ttft_ms": 799.1586847722829, "ttft_relative": 7.158676, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 10, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 10, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "766aff8f47d242d0", "is_multinode": true, "parallelism": {"attention_dp_size": 56, "moe_ep_size": 56, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 5700, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 37627.86573719315, "interactivity_tok_s": 32.983544202949645, "output_per_gpu": 2395.3919865937833, "total_per_gpu": 3728.0937731665035, "tpot_ms": 30.31814876675904, "tpot_relative": 1.0, "ttft_ms": 9722.544561971054, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "1b3dfd29893557c8", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 7500, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 44919.5021816004, "interactivity_tok_s": 22.401253650110032, "output_per_gpu": 4606.582783928396, "total_per_gpu": 4096.34403443608, "tpot_ms": 44.64035877720121, "tpot_relative": 1.0, "ttft_ms": 3827.281298628134, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 5, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "ed7500c5fab02bf7", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 28958.17981743453, "interactivity_tok_s": 67.51789119867811, "output_per_gpu": 13.982160486879192, "total_per_gpu": 26.369655963017387, "tpot_ms": 14.810889117632547, "tpot_relative": 1.0, "ttft_ms": 15289.078003025112, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 17715.339641201586, "interactivity_tok_s": 55.1258054648406, "output_per_gpu": 46.84426686090805, "total_per_gpu": 88.54775168144, "tpot_ms": 18.14032450986685, "tpot_relative": 1.224796, "ttft_ms": 1017.7764330031436, "ttft_relative": 0.066569, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 19719.819056507808, "interactivity_tok_s": 49.88531750566192, "output_per_gpu": 84.49862326089658, "total_per_gpu": 159.8246748692079, "tpot_ms": 20.045978456216126, "tpot_relative": 1.353462, "ttft_ms": 1298.986220915583, "ttft_relative": 0.084962, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 22992.048788635933, "interactivity_tok_s": 43.074669174489436, "output_per_gpu": 145.09539817288803, "total_per_gpu": 274.02669847835637, "tpot_ms": 23.21550041276325, "tpot_relative": 1.567462, "ttft_ms": 1613.416431195698, "ttft_relative": 0.105527, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 512, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 26931.857390313486, "interactivity_tok_s": 37.46823957183528, "output_per_gpu": 248.83344073269782, "total_per_gpu": 469.9594503726973, "tpot_ms": 26.68927100465366, "tpot_relative": 1.802003, "ttft_ms": 2344.097559514817, "ttft_relative": 0.153318, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 5, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "81693fb88d13ae7c", "is_multinode": true, "parallelism": {"attention_dp_size": 48, "moe_ep_size": 48, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 6500, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 41339.6292649441, "interactivity_tok_s": 29.904728701989725, "output_per_gpu": 2923.9771797570243, "total_per_gpu": 3900.1732877905324, "tpot_ms": 33.43952757322505, "tpot_relative": 1.0, "ttft_ms": 10557.322773214402, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "d75cad108130a847", "is_multinode": true, "parallelism": {"attention_dp_size": 40, "moe_ep_size": 40, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 7300, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 57881.09787464256, "interactivity_tok_s": 30.26424707327456, "output_per_gpu": 2817.1790654149027, "total_per_gpu": 3131.5217445883113, "tpot_ms": 33.04228906071381, "tpot_relative": 1.0, "ttft_ms": 27463.448558688484, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "6417dcc43dd31118", "is_multinode": true, "parallelism": {"attention_dp_size": 24, "moe_ep_size": 24, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 8192, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 53048.65426348572, "interactivity_tok_s": 18.77734954269691, "output_per_gpu": 5704.572861696831, "total_per_gpu": 3804.4621245673366, "tpot_ms": 53.255652387263076, "tpot_relative": 1.0, "ttft_ms": 4024.189195703616, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}]}, {"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.84, "tpot_shape_error_pct": 9.61, "ttft_mape_pct": 21.14, "ttft_shape_error_pct": 18.38}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.27, "tpot_shape_error_pct": 10.37, "ttft_mape_pct": 62.0, "ttft_shape_error_pct": 17.62}, "gpu": "h200", "precisions": ["fp8"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.84, "tpot_shape_error_pct": 9.61, "ttft_mape_pct": 21.14, "ttft_shape_error_pct": 18.38}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.27, "tpot_shape_error_pct": 10.37, "ttft_mape_pct": 62.0, "ttft_shape_error_pct": 17.62}, "framework": "sglang", "id": "166b0f15bfc66788", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 14069.272408610681, "interactivity_tok_s": 73.76550363277626, "output_per_gpu": 36.1859859692106, "total_per_gpu": null, "tpot_error_pct": 18.13, "tpot_ms": 13.556472209262726, "tpot_relative": 0.818667, "ttft_error_pct": 31.25, "ttft_ms": 201.0013385349126, "ttft_relative": 0.687473, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 12535.573363649986, "interactivity_tok_s": 73.70895770916827, "output_per_gpu": 35.58623708919092, "total_per_gpu": 71.52049398592276, "tpot_error_pct": 18.07, "tpot_ms": 13.566872074703278, "tpot_relative": 0.819295, "ttft_error_pct": 66.7, "ttft_ms": 97.36643737499969, "ttft_relative": 0.333017, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 15473.111001728103, "interactivity_tok_s": 60.389372717082374, "output_per_gpu": 28.75859739269279, "total_per_gpu": 57.798442884325254, "tpot_ms": 16.559204956224516, "tpot_relative": 1.0, "ttft_ms": 292.37696416676044, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 18194.24327122822, "interactivity_tok_s": 56.91916385662472, "output_per_gpu": 56.10736296686862, "total_per_gpu": null, "tpot_error_pct": 11.33, "tpot_ms": 17.56877529892969, "tpot_relative": 1.060967, "ttft_error_pct": 23.58, "ttft_ms": 221.38614042314717, "ttft_relative": 0.757194, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 16438.85629667504, "interactivity_tok_s": 56.852074285806275, "output_per_gpu": 55.495524204937304, "total_per_gpu": 110.58113743944016, "tpot_error_pct": 11.23, "tpot_ms": 17.589507727946888, "tpot_relative": 1.062219, "ttft_error_pct": 62.89, "ttft_ms": 107.5133200999993, "ttft_relative": 0.367722, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 18687.71014153026, "interactivity_tok_s": 50.470136680043815, "output_per_gpu": 48.7149179988705, "total_per_gpu": 97.07000915408763, "tpot_ms": 19.81369708466444, "tpot_relative": 1.196537, "ttft_ms": 289.68577765626833, "ttft_relative": 0.990795, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 25754.867361214354, "interactivity_tok_s": 40.1301923439022, "output_per_gpu": 79.48796953986749, "total_per_gpu": null, "tpot_error_pct": 1.26, "tpot_ms": 24.918893770314817, "tpot_relative": 1.504836, "ttft_error_pct": 20.73, "ttft_ms": 262.83903418229715, "ttft_relative": 0.898973, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 22965.151869487505, "interactivity_tok_s": 40.069338036096546, "output_per_gpu": 78.35620038836463, "total_per_gpu": 157.54652913909285, "tpot_error_pct": 1.11, "tpot_ms": 24.95673871874669, "tpot_relative": 1.507122, "ttft_error_pct": 62.71, "ttft_ms": 123.64784811250028, "ttft_relative": 0.422906, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 23433.219306298997, "interactivity_tok_s": 39.624906797939815, "output_per_gpu": 76.58441288913406, "total_per_gpu": 153.98409286101548, "tpot_ms": 25.23665241912928, "tpot_relative": 1.524026, "ttft_ms": 331.5692465112079, "ttft_relative": 1.134047, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 30862.323220883336, "interactivity_tok_s": 33.52363482134935, "output_per_gpu": 133.04088158159732, "total_per_gpu": null, "tpot_error_pct": 14.18, "tpot_ms": 29.82970090591595, "tpot_relative": 1.801397, "ttft_error_pct": 14.52, "ttft_ms": 346.5391941313206, "ttft_relative": 1.185248, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27951.752462512457, "interactivity_tok_s": 33.20303209686772, "output_per_gpu": 129.42067030718968, "total_per_gpu": 258.43194918956783, "tpot_error_pct": 13.35, "tpot_ms": 30.117731328951038, "tpot_relative": 1.818791, "ttft_error_pct": 62.03, "ttft_ms": 153.94795161874998, "ttft_relative": 0.526539, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 32503.466756551647, "interactivity_tok_s": 28.7705519629824, "output_per_gpu": 111.298971492806, "total_per_gpu": 222.24587523313207, "tpot_ms": 34.757762078622925, "tpot_relative": 2.098999, "ttft_ms": 405.42309284501243, "ttft_relative": 1.386645, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 41227.02320345255, "interactivity_tok_s": 25.062916870164024, "output_per_gpu": 199.3919457901315, "total_per_gpu": null, "tpot_error_pct": 14.31, "tpot_ms": 39.89958571783171, "tpot_relative": 2.409511, "ttft_error_pct": 15.62, "ttft_ms": 409.74701411070185, "ttft_relative": 1.401434, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 37708.91900384071, "interactivity_tok_s": 24.573828956499682, "output_per_gpu": 191.2958176444743, "total_per_gpu": 382.67788622332125, "tpot_error_pct": 12.6, "tpot_ms": 40.69369904747806, "tpot_relative": 2.457467, "ttft_error_pct": 55.7, "ttft_ms": 215.14525601093754, "ttft_relative": 0.735849, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 43409.8453538827, "interactivity_tok_s": 21.476679077113875, "output_per_gpu": 166.19612541450937, "total_per_gpu": 332.46718488289736, "tpot_ms": 46.562133577980724, "tpot_relative": 2.811858, "ttft_ms": 485.62459752429277, "ttft_relative": 1.660954, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 39}, {"aic": {"coverage_pct": 45.95, "points": 17, "status_counts": {"failed": 20, "success": 17, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 11.46, "tpot_shape_error_pct": 15.11, "ttft_mape_pct": 31.84, "ttft_shape_error_pct": 38.62}, "aisimulate": {"coverage_pct": 75.68, "points": 28, "status_counts": {"failed": 9, "success": 28, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 13.99, "tpot_shape_error_pct": 9.78, "ttft_mape_pct": 39.28, "ttft_shape_error_pct": 30.29}, "gpu_skus": ["b200", "b300", "gb300", "h200"], "gpus": [{"aic": {"coverage_pct": 37.5, "points": 6, "status_counts": {"failed": 10, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.55, "tpot_shape_error_pct": 18.18, "ttft_mape_pct": 25.14, "ttft_shape_error_pct": 42.13}, "aisimulate": {"coverage_pct": 100.0, "points": 16, "status_counts": {"failed": 0, "success": 16, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 14.96, "tpot_shape_error_pct": 12.76, "ttft_mape_pct": 38.69, "ttft_shape_error_pct": 33.46}, "gpu": "b200", "precisions": ["fp8"], "rows": 16, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 34.86, "tpot_shape_error_pct": null, "ttft_mape_pct": 57.94, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "e2f94bdafd8cfd03", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 13518.012742375027, "interactivity_tok_s": 102.53132726529547, "output_per_gpu": 10.767643336035782, "total_per_gpu": 97.22252580813375, "tpot_error_pct": 34.86, "tpot_ms": 9.753116697812194, "tpot_relative": 0.651446, "ttft_error_pct": 57.94, "ttft_ms": 4595.921882916677, "ttft_relative": 0.420629, "unavailable_metrics": {}}, "concurrency": 12, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 24621.615237434646, "interactivity_tok_s": 66.79364134152391, "output_per_gpu": 6.633080515696791, "total_per_gpu": 53.242875617033505, "tpot_ms": 14.971485008383956, "tpot_relative": 1.0, "ttft_ms": 10926.312594041763, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 19.59, "tpot_shape_error_pct": null, "ttft_mape_pct": 99.48, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "6590839908225470", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 42103.01787809142, "interactivity_tok_s": 36.383291083140506, "output_per_gpu": 194.68190261385527, "total_per_gpu": 1751.6808052057168, "tpot_error_pct": 19.59, "tpot_ms": 27.485144148034085, "tpot_relative": 0.804134, "ttft_error_pct": 99.48, "ttft_ms": 16777.736320103955, "ttft_relative": 1.994836, "unavailable_metrics": {}}, "concurrency": 224, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 39905.02235555847, "interactivity_tok_s": 29.257055043698664, "output_per_gpu": 309.7905290374925, "total_per_gpu": 1858.3205732831264, "tpot_ms": 34.179790088455206, "tpot_relative": 1.0, "ttft_ms": 8410.583223712016, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.07, "tpot_shape_error_pct": null, "ttft_mape_pct": 38.86, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "d60a715b47ac9c09", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 24157.036925763216, "interactivity_tok_s": 60.73979843639351, "output_per_gpu": 96.9737583806717, "total_per_gpu": 875.4870274941862, "tpot_error_pct": 22.07, "tpot_ms": 16.463670044068326, "tpot_relative": 0.779281, "ttft_error_pct": 38.86, "ttft_ms": 9026.177334142942, "ttft_relative": 1.388576, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 25917.213647487813, "interactivity_tok_s": 47.33339720547158, "output_per_gpu": 108.63332933912511, "total_per_gpu": 817.3905453809128, "tpot_ms": 21.126732054727807, "tpot_relative": 1.0, "ttft_ms": 6500.3139504667615, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.67, "tpot_shape_error_pct": null, "ttft_mape_pct": 34.71, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "5e51e9d438f6bbd2", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 15363.694096720323, "interactivity_tok_s": 72.40602706103688, "output_per_gpu": 57.63235577380196, "total_per_gpu": 519.4852591742275, "tpot_error_pct": 24.67, "tpot_ms": 13.811004975552924, "tpot_relative": 0.753336, "ttft_error_pct": 34.71, "ttft_ms": 2648.689394695314, "ttft_relative": 0.652887, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 20936.15391340948, "interactivity_tok_s": 54.54603664491377, "output_per_gpu": 47.70749815526209, "total_per_gpu": 376.31698393484555, "tpot_ms": 18.333137685325237, "tpot_relative": 1.0, "ttft_ms": 4056.886126256222, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 64}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 5.69, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "e316739ae34eec94", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 47078.35225668026, "interactivity_tok_s": 48.256089215380825, "output_per_gpu": 149.91793454833797, "total_per_gpu": 1348.8734776204246, "tpot_error_pct": 18.05, "tpot_ms": 20.72277335895812, "tpot_relative": 0.81951, "ttft_error_pct": 5.69, "ttft_ms": 27991.56216511631, "ttft_relative": 1.056909, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 49773.96583048958, "interactivity_tok_s": 39.54636857419325, "output_per_gpu": 189.3244940740268, "total_per_gpu": 1277.6810000570993, "tpot_ms": 25.286771859314772, "tpot_relative": 1.0, "ttft_ms": 26484.37344547783, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 32}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.05, "tpot_shape_error_pct": null, "ttft_mape_pct": 17.51, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "04fb41078deeac4d", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 36972.01729403236, "interactivity_tok_s": 55.434190988716956, "output_per_gpu": 119.22905702773778, "total_per_gpu": 1074.4433958016164, "tpot_error_pct": 21.05, "tpot_ms": 18.039408209340323, "tpot_relative": 0.789487, "ttft_error_pct": 17.51, "ttft_ms": 20376.802169179424, "ttft_relative": 1.17514, "unavailable_metrics": {}}, "concurrency": 200, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 38359.16128801927, "interactivity_tok_s": 43.76458479579884, "output_per_gpu": 143.56130207089515, "total_per_gpu": 1035.0616161703713, "tpot_ms": 22.849525584805605, "tpot_relative": 1.0, "ttft_ms": 17339.889399309875, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 27.77, "tpot_shape_error_pct": null, "ttft_mape_pct": 15.53, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "b48fba0edd20fb08", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 60249.10888646322, "interactivity_tok_s": 17.56206907403414, "output_per_gpu": 341.9086145003378, "total_per_gpu": 3077.2652359373355, "tpot_error_pct": 27.77, "tpot_ms": 56.94089892167201, "tpot_relative": 1.277725, "ttft_error_pct": 15.53, "ttft_ms": 7843.858043665359, "ttft_relative": 0.844698, "unavailable_metrics": {}}, "concurrency": 560, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 50347.60135752757, "interactivity_tok_s": 22.439495759399627, "output_per_gpu": 1227.3495074861712, "total_per_gpu": 3682.156792660731, "tpot_ms": 44.56428124420364, "tpot_relative": 1.0, "ttft_ms": 9285.987414501591, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 20.33, "tpot_shape_error_pct": null, "ttft_mape_pct": 106.75, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "acb4d737439366c9", "is_multinode": true, "parallelism": {"attention_dp_size": 8, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 45243.435853769304, "interactivity_tok_s": 32.09340297407203, "output_per_gpu": 292.01235223072496, "total_per_gpu": 2626.9269377859, "tpot_error_pct": 20.33, "tpot_ms": 31.159051622163314, "tpot_relative": 0.796736, "ttft_error_pct": 106.75, "ttft_ms": 16523.094292841753, "ttft_relative": 2.067531, "unavailable_metrics": {}}, "concurrency": 240, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 44039.02575415133, "interactivity_tok_s": 25.56998314738108, "output_per_gpu": 602.8117204307092, "total_per_gpu": 2711.7166422559217, "tpot_ms": 39.10835584975431, "tpot_relative": 1.0, "ttft_ms": 7991.704738947931, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 16}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 12.66, "tpot_shape_error_pct": null, "ttft_mape_pct": 22.44, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "3ef6bde4ba09c64a", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 48075.73609873745, "interactivity_tok_s": 31.18649064756421, "output_per_gpu": 197.72280244294575, "total_per_gpu": 1778.9935869886167, "tpot_error_pct": 12.66, "tpot_ms": 32.06516601373691, "tpot_relative": 1.126568, "ttft_error_pct": 22.44, "ttft_ms": 18564.766773068135, "ttft_relative": 0.775602, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26067557908", "measured": {"e2e_ms": 50150.39835041025, "interactivity_tok_s": 35.1337045309968, "output_per_gpu": 282.40427453737055, "total_per_gpu": 1694.0817940990835, "tpot_ms": 28.46269738272966, "tpot_relative": 1.0, "ttft_ms": 23935.936583711828, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 85.71, "points": 6, "status_counts": {"failed": 1, "success": 6, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.55, "tpot_shape_error_pct": 18.18, "ttft_mape_pct": 25.14, "ttft_shape_error_pct": 42.13}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 5.48, "tpot_shape_error_pct": 12.76, "ttft_mape_pct": 31.45, "ttft_shape_error_pct": 33.46}, "framework": "sglang", "id": "a38ec62224dcca33", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12620.318799948625, "interactivity_tok_s": 85.95878798400773, "output_per_gpu": 39.558030835168054, "total_per_gpu": null, "tpot_error_pct": 18.85, "tpot_ms": 11.633481851629247, "tpot_relative": 0.811543, "ttft_error_pct": 59.93, "ttft_ms": 719.2668657319066, "ttft_relative": 1.599251, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11587.89914805, "interactivity_tok_s": 81.87906832611719, "output_per_gpu": 38.80061364524287, "total_per_gpu": 348.7943765139745, "tpot_error_pct": 14.8, "tpot_ms": 12.213133593766445, "tpot_relative": 0.85198, "ttft_error_pct": 19.55, "ttft_ms": 361.8159080999996, "ttft_relative": 0.804478, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 13625.228520225392, "interactivity_tok_s": 69.75929282360671, "output_per_gpu": 32.95170785509568, "total_per_gpu": 296.2162016681886, "tpot_ms": 14.335007703254663, "tpot_relative": 1.0, "ttft_ms": 449.75245357491076, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16950.855865602338, "interactivity_tok_s": 63.30281381478303, "output_per_gpu": 59.536486201391966, "total_per_gpu": null, "tpot_error_pct": 9.27, "tpot_ms": 15.797086096771123, "tpot_relative": 1.101994, "ttft_error_pct": 10.68, "ttft_ms": 790.436788605479, "ttft_relative": 1.757493, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15482.549677174999, "interactivity_tok_s": 61.55715031339589, "output_per_gpu": 58.95614123247735, "total_per_gpu": 524.4421339963036, "tpot_error_pct": 6.69, "tpot_ms": 16.245066493638237, "tpot_relative": 1.133244, "ttft_error_pct": 39.12, "ttft_ms": 434.82364033750025, "ttft_relative": 0.966807, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 16830.750811187863, "interactivity_tok_s": 57.437618292667146, "output_per_gpu": 53.22069708116762, "total_per_gpu": 473.4227065499137, "tpot_ms": 17.410192652916223, "tpot_relative": 1.214523, "ttft_ms": 714.1942818122516, "ttft_relative": 1.587972, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 22237.58961837046, "interactivity_tok_s": 48.01695519853593, "output_per_gpu": 91.67030857797192, "total_per_gpu": null, "tpot_error_pct": 6.89, "tpot_ms": 20.825976904726577, "tpot_relative": 1.452805, "ttft_error_pct": 8.23, "ttft_ms": 932.6152448351722, "ttft_relative": 2.073619, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 20003.084365793708, "interactivity_tok_s": 47.09913646112772, "output_per_gpu": 89.22787606369398, "total_per_gpu": 805.2592479981487, "tpot_error_pct": 5.08, "tpot_ms": 21.23181177271326, "tpot_relative": 1.481116, "ttft_error_pct": 33.63, "ttft_ms": 571.925834825, "ttft_relative": 1.271646, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 21342.752730300297, "interactivity_tok_s": 44.707426511798744, "output_per_gpu": 82.2276383123265, "total_per_gpu": 742.0838544311144, "tpot_ms": 22.367648465207225, "tpot_relative": 1.560351, "ttft_ms": 861.6933357189282, "ttft_relative": 1.915928, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31818.772803542663, "interactivity_tok_s": 33.43482250123257, "output_per_gpu": 129.4954218465671, "total_per_gpu": null, "tpot_error_pct": 2.55, "tpot_ms": 29.90893700611496, "tpot_relative": 2.086426, "ttft_error_pct": 7.95, "ttft_ms": 1221.930246287059, "ttft_relative": 2.716895, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 28487.363421515725, "interactivity_tok_s": 33.48041640526677, "output_per_gpu": 127.35494859249758, "total_per_gpu": 1138.8485010194233, "tpot_error_pct": 2.69, "tpot_ms": 29.86820677184561, "tpot_relative": 2.083585, "ttft_error_pct": 26.68, "ttft_ms": 829.9829810562487, "ttft_relative": 1.845422, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 29548.45503028748, "interactivity_tok_s": 32.58078710461137, "output_per_gpu": 121.84771583676091, "total_per_gpu": 1089.601072176244, "tpot_ms": 30.692935587749, "tpot_relative": 2.141117, "ttft_ms": 1131.9242276250634, "ttft_relative": 2.516772, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 48378.07286125501, "interactivity_tok_s": 21.787391444985744, "output_per_gpu": 170.74829997201985, "total_per_gpu": null, "tpot_error_pct": 0.75, "tpot_ms": 45.8981059079537, "tpot_relative": 3.201819, "ttft_error_pct": 16.15, "ttft_ms": 1424.3105174183697, "ttft_relative": 3.166877, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43779.00270794983, "interactivity_tok_s": 21.743320383544372, "output_per_gpu": 166.1012037098343, "total_per_gpu": 1497.1993717733485, "tpot_error_pct": 0.55, "tpot_ms": 45.99113577688958, "tpot_relative": 3.208309, "ttft_error_pct": 20.56, "ttft_ms": 1349.4115722984366, "ttft_relative": 3.000343, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 44357.43064428744, "interactivity_tok_s": 21.62412847454161, "output_per_gpu": 163.35841770074526, "total_per_gpu": 1472.4765076519632, "tpot_ms": 46.24463830657101, "tpot_relative": 3.225993, "ttft_ms": 1698.670336810892, "ttft_relative": 3.776901, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 77454.58510075486, "interactivity_tok_s": 13.460063115961619, "output_per_gpu": 212.7288895082062, "total_per_gpu": null, "tpot_error_pct": 0.99, "tpot_ms": 74.29385667695345, "tpot_relative": 5.182687, "ttft_error_pct": 47.93, "ttft_ms": 1451.9697202314826, "ttft_relative": 3.228375, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 72126.80783200337, "interactivity_tok_s": 13.608915329088733, "output_per_gpu": 200.34161146024474, "total_per_gpu": 1808.7004652558041, "tpot_error_pct": 2.07, "tpot_ms": 73.48124195192278, "tpot_relative": 5.125999, "ttft_error_pct": 57.0, "ttft_ms": 4377.702380984328, "ttft_relative": 9.733582, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 71917.4949711045, "interactivity_tok_s": 13.32702332593489, "output_per_gpu": 202.07635149920574, "total_per_gpu": 1824.361840307633, "tpot_ms": 75.03551059702599, "tpot_relative": 5.234424, "ttft_ms": 2788.281315259246, "ttft_relative": 6.199591, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 142368.36348432803, "interactivity_tok_s": 13.053417995215762, "output_per_gpu": 201.88099768105204, "total_per_gpu": 1816.4065842283876, "tpot_error_pct": 6.5, "tpot_ms": 76.60828760455784, "tpot_relative": 5.34414, "ttft_error_pct": 23.62, "ttft_ms": 71590.77601552031, "ttft_relative": 159.178178, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "27431570825", "measured": {"e2e_ms": 133652.18128821912, "interactivity_tok_s": 12.20441718956953, "output_per_gpu": 215.9336474964013, "total_per_gpu": 1942.8440693986522, "tpot_ms": 81.93754642004923, "tpot_relative": 5.715905, "ttft_ms": 57912.77874507307, "ttft_relative": 128.765898, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.54, "tpot_shape_error_pct": 8.97, "ttft_mape_pct": 32.45, "ttft_shape_error_pct": 45.77}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.52, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 39.89, "ttft_shape_error_pct": 10.16}, "gpu": "b300", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.54, "tpot_shape_error_pct": 8.97, "ttft_mape_pct": 32.45, "ttft_shape_error_pct": 45.77}, "aisimulate": {"points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.52, "tpot_shape_error_pct": 4.76, "ttft_mape_pct": 39.89, "ttft_shape_error_pct": 10.16}, "framework": "sglang", "id": "38f487e805399eb4", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 12445.933533044425, "interactivity_tok_s": 87.04412299865965, "output_per_gpu": 40.13581146212449, "total_per_gpu": null, "tpot_error_pct": 23.43, "tpot_ms": 11.488426392846748, "tpot_relative": 0.765665, "ttft_error_pct": 21.05, "ttft_ms": 693.2733331622028, "ttft_relative": 1.210488, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 11410.551539275011, "interactivity_tok_s": 83.0939185348644, "output_per_gpu": 39.42058110967854, "total_per_gpu": 354.3675142791113, "tpot_error_pct": 19.79, "tpot_ms": 12.03457506436433, "tpot_relative": 0.802064, "ttft_error_pct": 39.13, "ttft_ms": 348.63468257499943, "ttft_relative": 0.608733, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 14362.508092535427, "interactivity_tok_s": 66.64667161646844, "output_per_gpu": 31.283853743267994, "total_per_gpu": 281.22318788830137, "tpot_ms": 15.004500236031282, "tpot_relative": 1.0, "ttft_ms": 572.7221262059174, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 16105.4769420913, "interactivity_tok_s": 66.67137797995765, "output_per_gpu": 62.647997373730604, "total_per_gpu": null, "tpot_error_pct": 19.18, "tpot_ms": 14.99894002941733, "tpot_relative": 0.999629, "ttft_error_pct": 9.62, "ttft_ms": 761.5612919973721, "ttft_relative": 1.329722, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 14729.000431637518, "interactivity_tok_s": 64.7268576192866, "output_per_gpu": 61.95086554637944, "total_per_gpu": 551.081591346817, "tpot_error_pct": 16.75, "tpot_ms": 15.449537282990715, "tpot_relative": 1.02966, "ttft_error_pct": 50.32, "ttft_ms": 418.58386447500004, "ttft_relative": 0.730867, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 18027.290700847516, "interactivity_tok_s": 53.88616887503096, "output_per_gpu": 49.893790878206886, "total_per_gpu": 443.82833771552464, "tpot_ms": 18.557637718115203, "tpot_relative": 1.236805, "ttft_ms": 842.614668168244, "ttft_relative": 1.471245, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 21466.886012660845, "interactivity_tok_s": 49.73668221331928, "output_per_gpu": 94.96247393469673, "total_per_gpu": null, "tpot_error_pct": 17.31, "tpot_ms": 20.105884741387197, "tpot_relative": 1.33999, "ttft_error_pct": 12.62, "ttft_ms": 898.5659222217423, "ttft_relative": 1.568939, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 19290.756139925048, "interactivity_tok_s": 48.834292306897254, "output_per_gpu": 92.50393997937955, "total_per_gpu": 834.8249048479843, "tpot_error_pct": 15.79, "tpot_ms": 20.477413570683854, "tpot_relative": 1.364751, "ttft_error_pct": 46.51, "ttft_ms": 550.0538914562509, "ttft_relative": 0.96042, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 23301.24281329263, "interactivity_tok_s": 41.12490388624421, "output_per_gpu": 75.56793696587174, "total_per_gpu": 681.98171668315, "tpot_ms": 24.316166252111, "tpot_relative": 1.620592, "ttft_ms": 1028.3463893858425, "ttft_relative": 1.795542, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 31181.467432022284, "interactivity_tok_s": 34.09525436272808, "output_per_gpu": 132.13107214098557, "total_per_gpu": null, "tpot_error_pct": 13.94, "tpot_ms": 29.32959494483697, "tpot_relative": 1.95472, "ttft_error_pct": 14.36, "ttft_ms": 1177.2918034540626, "ttft_relative": 2.055607, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 27992.121773512412, "interactivity_tok_s": 34.05221435174375, "output_per_gpu": 129.6361669272735, "total_per_gpu": 1159.2478817248436, "tpot_error_pct": 13.83, "tpot_ms": 29.36666584059582, "tpot_relative": 1.957191, "ttft_error_pct": 41.79, "ttft_ms": 800.2368428156249, "ttft_relative": 1.397251, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 32940.81332381593, "interactivity_tok_s": 29.34126957304454, "output_per_gpu": 109.55812098681331, "total_per_gpu": 979.7036019351594, "tpot_ms": 34.081688166577756, "tpot_relative": 2.271431, "ttft_ms": 1374.716215286935, "ttft_relative": 2.40032, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 47536.73322380698, "interactivity_tok_s": 22.160442439584347, "output_per_gpu": 173.74640239664816, "total_per_gpu": null, "tpot_error_pct": 14.71, "tpot_ms": 45.12545283002736, "tpot_relative": 3.007461, "ttft_error_pct": 33.56, "ttft_ms": 1373.3949786889898, "ttft_relative": 2.398013, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 43040.031884935976, "interactivity_tok_s": 22.10178387439194, "output_per_gpu": 168.96960023976354, "total_per_gpu": 1523.0544612410276, "tpot_error_pct": 14.49, "tpot_ms": 45.24521666138642, "tpot_relative": 3.015443, "ttft_error_pct": 37.1, "ttft_ms": 1300.3231980921887, "ttft_relative": 2.270426, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 50891.91293434269, "interactivity_tok_s": 18.899974347018833, "output_per_gpu": 142.72900920274233, "total_per_gpu": 1286.5275996764267, "tpot_ms": 52.91012472499646, "tpot_relative": 3.526284, "ttft_ms": 2067.1420337820564, "ttft_relative": 3.609328, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 76094.55877012054, "interactivity_tok_s": 13.695871337344187, "output_per_gpu": 216.5125177208428, "total_per_gpu": null, "tpot_error_pct": 16.18, "tpot_ms": 73.01470460468808, "tpot_relative": 4.866187, "ttft_error_pct": 59.01, "ttft_ms": 1400.5159595246319, "ttft_relative": 2.445367, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 69493.48366132569, "interactivity_tok_s": 13.707932746079326, "output_per_gpu": 209.48711328604196, "total_per_gpu": 1891.2668042542289, "tpot_error_pct": 16.26, "tpot_ms": 72.95046003825887, "tpot_relative": 4.861905, "ttft_error_pct": 32.95, "ttft_ms": 2290.8213196828137, "ttft_relative": 3.999883, "unavailable_metrics": {}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 83687.5552453334, "interactivity_tok_s": 11.479616118493489, "output_per_gpu": 173.96650379640982, "total_per_gpu": 1570.5838345916022, "tpot_ms": 87.11092685312143, "tpot_relative": 5.805653, "ttft_ms": 3416.6902000722985, "ttft_relative": 5.965703, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 129977.0603292693, "interactivity_tok_s": 7.959174407277782, "output_per_gpu": 253.05292511249505, "total_per_gpu": null, "tpot_error_pct": 18.04, "tpot_ms": 125.64117191421398, "tpot_relative": 8.373566, "ttft_error_pct": 76.93, "ttft_ms": 1446.141461028396, "ttft_relative": 2.525032, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 119383.52660210527, "interactivity_tok_s": 8.026516443106322, "output_per_gpu": 245.1400829289419, "total_per_gpu": 2205.626412615134, "tpot_error_pct": 18.73, "tpot_ms": 124.58704932435077, "tpot_relative": 8.303312, "ttft_error_pct": 31.45, "ttft_ms": 4296.68427006564, "ttft_relative": 7.502215, "unavailable_metrics": {}}, "concurrency": 256, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26008965342", "measured": {"e2e_ms": 147920.14752996352, "interactivity_tok_s": 6.523310030615168, "output_per_gpu": 197.99715657733336, "total_per_gpu": 1781.462064268976, "tpot_ms": 153.2964086187541, "tpot_relative": 10.216695, "ttft_ms": 6268.3630821245515, "ttft_relative": 10.944859, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 7, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 9, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "gpu": "gb300", "precisions": ["fp8"], "rows": 9, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "931f552f8fbe9ba9", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1300, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 30671.96364123543, "interactivity_tok_s": 38.422676802034005, "output_per_gpu": 1161.9513347074612, "total_per_gpu": 4644.701508384018, "tpot_ms": 26.026297052449568, "tpot_relative": 1.0, "ttft_ms": 6697.436154584025, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 4, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "8bead245cdadac90", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 24, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 26934.662596320744, "interactivity_tok_s": 64.72380985499458, "output_per_gpu": 10.079908262641567, "total_per_gpu": 85.66950762088055, "tpot_ms": 15.450264782625933, "tpot_relative": 1.0, "ttft_ms": 12758.016869004283, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 17579.893598375045, "interactivity_tok_s": 58.19631040372942, "output_per_gpu": 23.770207968836157, "total_per_gpu": 200.77609485043266, "tpot_ms": 17.183219916565648, "tpot_relative": 1.112163, "ttft_ms": 1699.0687382404758, "ttft_relative": 0.133177, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 20438.999512101494, "interactivity_tok_s": 53.91698239181216, "output_per_gpu": 40.61576694236417, "total_per_gpu": 345.80414828572907, "tpot_ms": 18.547032041464178, "tpot_relative": 1.200435, "ttft_ms": 3364.8021687186997, "ttft_relative": 0.26374, "unavailable_metrics": {}}, "status": "failed"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 128, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 29413.686483857113, "interactivity_tok_s": 51.11313251071148, "output_per_gpu": 56.234017711182624, "total_per_gpu": 479.5382961834895, "tpot_ms": 19.564443634724125, "tpot_relative": 1.266285, "ttft_ms": 11432.732567741414, "ttft_relative": 0.896121, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 4, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "35220d3c16514791", "is_multinode": true, "parallelism": {"attention_dp_size": 40, "moe_ep_size": 40, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 900, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 27690.84134597946, "interactivity_tok_s": 41.586722409960046, "output_per_gpu": 709.6528005550585, "total_per_gpu": 3548.0822198602423, "tpot_ms": 24.04613641205106, "tpot_relative": 1.0, "ttft_ms": 5544.715905691241, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "40fc29af48097e1b", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 2800, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 46547.940823650046, "interactivity_tok_s": 26.20907723829772, "output_per_gpu": 3310.7822310608053, "total_per_gpu": 6620.7990466272995, "tpot_ms": 38.15471986700704, "tpot_relative": 1.0, "ttft_ms": 11413.627833195695, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "242a60f5241e8c65", "is_multinode": true, "parallelism": {"attention_dp_size": 24, "moe_ep_size": 24, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 1700, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 33676.603961944944, "interactivity_tok_s": 33.953620919046166, "output_per_gpu": 1849.8299996499702, "total_per_gpu": 5545.736504192244, "tpot_ms": 29.451939820623178, "tpot_relative": 1.0, "ttft_ms": 6537.855990437715, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "ef73103ce75b05f8", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 4, "pp_size": 1, "tp_size": 4}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": "ValueError: no KV budget: non-KV memory (220311846912 bytes) meets/exceeds the KV-cache memory limit (capacity=298013687808 bytes)", "aisimulate": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "concurrency": 150, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26606969606", "measured": {"e2e_ms": 34388.21149378873, "interactivity_tok_s": 44.21025883223157, "output_per_gpu": 106.66640941833761, "total_per_gpu": 866.5523858089797, "tpot_ms": 22.619184470165287, "tpot_relative": 1.0, "ttft_ms": 13605.393962935326, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "failed"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 40}]}, {"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.17, "tpot_shape_error_pct": 22.27, "ttft_mape_pct": 40.81, "ttft_shape_error_pct": 18.49}, "aisimulate": {"coverage_pct": 100.0, "points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.35, "tpot_shape_error_pct": 12.85, "ttft_mape_pct": 40.31, "ttft_shape_error_pct": 55.74}, "gpu": "h200", "precisions": ["fp8"], "rows": 5, "topologies": [{"aic": {"coverage_pct": 80.0, "points": 4, "status_counts": {"failed": 1, "success": 4, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 8.17, "tpot_shape_error_pct": 22.27, "ttft_mape_pct": 40.81, "ttft_shape_error_pct": 18.49}, "aisimulate": {"points": 5, "status_counts": {"failed": 0, "success": 5, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.35, "tpot_shape_error_pct": 12.85, "ttft_mape_pct": 40.31, "ttft_shape_error_pct": 55.74}, "framework": "sglang", "id": "6d34df651c1696f3", "is_multinode": false, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": 16596.326805952744, "interactivity_tok_s": 67.19512348314322, "output_per_gpu": 29.734272906623964, "total_per_gpu": null, "tpot_error_pct": 20.52, "tpot_ms": 14.88203232859395, "tpot_relative": 0.794826, "ttft_error_pct": 63.47, "ttft_ms": 1372.0077338011322, "ttft_relative": 1.634689, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 15412.56939307494, "interactivity_tok_s": 62.4485503927248, "output_per_gpu": 29.23560484127452, "total_per_gpu": 262.81065180708816, "tpot_error_pct": 14.48, "tpot_ms": 16.01318195076149, "tpot_relative": 0.855239, "ttft_error_pct": 17.41, "ttft_ms": 693.1617769749996, "ttft_relative": 0.825873, "unavailable_metrics": {}}, "concurrency": 4, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 18050.4988046363, "interactivity_tok_s": 53.40843101070808, "output_per_gpu": 24.907906697328077, "total_per_gpu": 223.9072264731553, "tpot_ms": 18.72363559602614, "tpot_relative": 1.0, "ttft_ms": 839.3082645488903, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 23238.550468776248, "interactivity_tok_s": 47.07368929471838, "output_per_gpu": 43.167679370906654, "total_per_gpu": null, "tpot_error_pct": 8.99, "tpot_ms": 21.243289297747882, "tpot_relative": 1.134571, "ttft_error_pct": 49.13, "ttft_ms": 1506.6655171801656, "ttft_relative": 1.795128, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 21351.83874068747, "interactivity_tok_s": 45.15747682577615, "output_per_gpu": 42.81024138105443, "total_per_gpu": 380.8168899359568, "tpot_error_pct": 5.13, "tpot_ms": 22.14472708158916, "tpot_relative": 1.182715, "ttft_error_pct": 17.96, "ttft_ms": 828.8376480125, "ttft_relative": 0.987525, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 22637.175355199724, "interactivity_tok_s": 42.84261558834118, "output_per_gpu": 40.28548244343035, "total_per_gpu": 358.35799189083826, "tpot_ms": 23.341245306043625, "tpot_relative": 1.246619, "ttft_ms": 1010.3125886293127, "ttft_relative": 1.203744, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 35912.36543059787, "interactivity_tok_s": 29.969204572936054, "output_per_gpu": 56.70673343998788, "total_per_gpu": null, "tpot_error_pct": 1.87, "tpot_ms": 33.367585634990746, "tpot_relative": 1.78211, "ttft_error_pct": 29.32, "ttft_ms": 1777.3253260023307, "ttft_relative": 2.117607, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 32313.994179750058, "interactivity_tok_s": 29.31196341756132, "output_per_gpu": 55.55083596431508, "total_per_gpu": 501.33239036599974, "tpot_error_pct": 4.15, "tpot_ms": 34.11576310172666, "tpot_relative": 1.822069, "ttft_error_pct": 21.1, "ttft_ms": 1084.3528994750006, "ttft_relative": 1.29196, "unavailable_metrics": {}}, "concurrency": 16, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 31366.8967634032, "interactivity_tok_s": 30.529518223595073, "output_per_gpu": 56.95101132788558, "total_per_gpu": 513.9686225624208, "tpot_ms": 32.75518443088758, "tpot_relative": 1.749403, "ttft_ms": 1374.3290659971535, "ttft_relative": 1.637454, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 52555.04044742673, "interactivity_tok_s": 20.366820836445612, "output_per_gpu": 78.46082918003462, "total_per_gpu": null, "tpot_error_pct": 1.32, "tpot_ms": 49.099464665125346, "tpot_relative": 2.622325, "ttft_error_pct": 21.3, "ttft_ms": 2326.2880950034973, "ttft_relative": 2.771673, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 49521.847864000054, "interactivity_tok_s": 20.424115559780013, "output_per_gpu": 73.24966334066688, "total_per_gpu": 655.0218127967595, "tpot_error_pct": 1.6, "tpot_ms": 48.96172845639593, "tpot_relative": 2.614969, "ttft_error_pct": 116.64, "ttft_ms": 4154.712807528119, "ttft_relative": 4.950163, "unavailable_metrics": {}}, "concurrency": 32, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 48011.179732804885, "interactivity_tok_s": 20.097288541487142, "output_per_gpu": 75.67440175157569, "total_per_gpu": 676.7045957207548, "tpot_ms": 49.75795605141881, "tpot_relative": 2.657494, "ttft_ms": 1917.7975298196543, "ttft_relative": 2.284974, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 97429.39807775624, "interactivity_tok_s": 19.383079405096527, "output_per_gpu": 73.32496655777774, "total_per_gpu": 660.9349686434982, "tpot_error_pct": 11.38, "tpot_ms": 51.59138953622937, "tpot_relative": 2.755415, "ttft_error_pct": 28.44, "ttft_ms": 49847.84223683433, "ttft_relative": 59.391578, "unavailable_metrics": {}}, "concurrency": 64, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "26118912054", "measured": {"e2e_ms": 92533.12720437534, "interactivity_tok_s": 17.178032209774482, "output_per_gpu": 77.39555921875989, "total_per_gpu": 697.6263871199968, "tpot_ms": 58.21388548980537, "tpot_relative": 3.109112, "ttft_ms": 38810.78893990052, "ttft_relative": 46.2414, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 5, "serving": "aggregated", "spec_method": "none", "total_gpus": 8}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 37}]}, {"aic": {"coverage_pct": 7.14, "points": 1, "status_counts": {"failed": 13, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 58.5, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 14, "status_counts": {"failed": 0, "success": 14, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 24.79, "tpot_shape_error_pct": 15.85, "ttft_mape_pct": 67.57, "ttft_shape_error_pct": 246.59}, "frameworks": ["dynamo-sglang"], "gpu_skus": ["gb200"], "hf_model_paths": ["zai-org/GLM-5.1-FP8"], "model": "zai-org/GLM-5.1-FP8", "precisions": ["fp8"], "rows": 14, "workloads": [{"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 58.5, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.61, "tpot_shape_error_pct": 22.15, "ttft_mape_pct": 72.16, "ttft_shape_error_pct": 93.6}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 14.29, "points": 1, "status_counts": {"failed": 6, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 58.5, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 32.61, "tpot_shape_error_pct": 22.15, "ttft_mape_pct": 72.16, "ttft_shape_error_pct": 93.6}, "gpu": "gb200", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 31.08, "tpot_shape_error_pct": 41.39, "ttft_mape_pct": 72.94, "ttft_shape_error_pct": 162.7}, "framework": "dynamo-sglang", "id": "274cf3aba3189be2", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 29992.713418230793, "interactivity_tok_s": 34.89285814466228, "output_per_gpu": 418.5239553534263, "total_per_gpu": 837.0479107068526, "tpot_error_pct": 2.73, "tpot_ms": 28.659159873178076, "tpot_relative": 1.027348, "ttft_error_pct": 87.02, "ttft_ms": 674.39286796913, "ttft_relative": 0.12982, "unavailable_metrics": {}}, "concurrency": 298, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 33732.71567269272, "interactivity_tok_s": 35.8470989996744, "output_per_gpu": 543.0159846510968, "total_per_gpu": 724.021312868129, "tpot_ms": 27.89626016903301, "tpot_relative": 1.0, "ttft_ms": 5194.841519771956, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 45167.654655097336, "interactivity_tok_s": 23.642807764509488, "output_per_gpu": 1072.7097480761558, "total_per_gpu": 2145.4194961523117, "tpot_error_pct": 36.83, "tpot_ms": 42.29616084351506, "tpot_relative": 1.516195, "ttft_error_pct": 74.59, "ttft_ms": 1898.6821121879846, "ttft_relative": 0.365494, "unavailable_metrics": {}}, "concurrency": 1170, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 39096.13718643617, "interactivity_tok_s": 32.350336244972944, "output_per_gpu": 1825.78254798622, "total_per_gpu": 2434.3767306482932, "tpot_ms": 30.911579787842058, "tpot_relative": 1.10809, "ttft_ms": 7473.591063473748, "ttft_relative": 1.438656, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 57509.47617230163, "interactivity_tok_s": 18.945267830397047, "output_per_gpu": 1407.161826381702, "total_per_gpu": 2814.323652763404, "tpot_error_pct": 53.67, "tpot_ms": 52.783629608842666, "tpot_relative": 1.89214, "ttft_error_pct": 57.2, "ttft_ms": 3511.8230824749267, "ttft_relative": 0.676021, "unavailable_metrics": {}}, "concurrency": 1955, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 43342.768549677385, "interactivity_tok_s": 29.11391576980834, "output_per_gpu": 2773.287908426282, "total_per_gpu": 3697.717211235042, "tpot_ms": 34.34783585645385, "tpot_relative": 1.23127, "ttft_ms": 8204.932468525109, "ttft_relative": 1.579438, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 24}, {"aic": {"coverage_pct": 33.33, "points": 1, "status_counts": {"failed": 2, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 6.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 58.5, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 17.18, "tpot_shape_error_pct": 2.92, "ttft_mape_pct": 64.07, "ttft_shape_error_pct": 24.51}, "framework": "dynamo-sglang", "id": "3ca9943386196f28", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 17047.80127827498, "interactivity_tok_s": 79.76726247658719, "output_per_gpu": 8.129850255738392, "total_per_gpu": 16.259700511476783, "tpot_error_pct": 18.76, "tpot_ms": 12.536471341153446, "tpot_relative": 0.8124, "ttft_error_pct": 59.8, "ttft_ms": 4222.991096274995, "ttft_relative": 0.401979, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 26291.827114811167, "interactivity_tok_s": 64.80294214682081, "output_per_gpu": 6.239347725954677, "total_per_gpu": 10.696024673065162, "tpot_ms": 15.43139812594233, "tpot_relative": 1.0, "ttft_ms": 10505.506831972161, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14581.765887879994, "interactivity_tok_s": 71.79881897591291, "output_per_gpu": 24.60763971995965, "total_per_gpu": 49.2152794399193, "tpot_error_pct": 18.56, "tpot_ms": 13.927805697409596, "tpot_relative": 0.902563, "ttft_error_pct": 76.05, "ttft_ms": 333.6206594299997, "ttft_relative": 0.031757, "unavailable_metrics": {}}, "concurrency": 20, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 18888.23399020126, "interactivity_tok_s": 58.472995251482544, "output_per_gpu": 21.482486409001833, "total_per_gpu": 36.82711955828886, "tpot_ms": 17.101911672203002, "tpot_relative": 1.108254, "ttft_ms": 1392.9783495375887, "ttft_relative": 0.132595, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": 19806.962, "interactivity_tok_s": 52.78437582475587, "output_per_gpu": 62.48008928571429, "total_per_gpu": null, "tpot_error_pct": 6.94, "tpot_ms": 18.945, "tpot_relative": 1.227692, "ttft_error_pct": 58.5, "ttft_ms": 426.227, "ttft_relative": 0.040572, "unavailable_metrics": {"total_per_gpu": "unsupported_by_predictor"}}, "aic_status": "success", "aisim_error": null, "aisimulate": {"e2e_ms": 18313.10005807222, "interactivity_tok_s": 57.263526643964866, "output_per_gpu": 71.31821186778751, "total_per_gpu": 142.63642373557502, "tpot_error_pct": 14.22, "tpot_ms": 17.463122839385797, "tpot_relative": 1.131662, "ttft_error_pct": 56.35, "ttft_ms": 448.3253933805549, "ttft_relative": 0.042675, "unavailable_metrics": {}}, "concurrency": 72, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 21853.463794609223, "interactivity_tok_s": 49.12015089243087, "output_per_gpu": 69.20193122982461, "total_per_gpu": 118.63188210827074, "tpot_ms": 20.358243650145102, "tpot_relative": 1.319274, "ttft_ms": 1026.9805405107843, "ttft_relative": 0.097756, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 83.48, "tpot_shape_error_pct": null, "ttft_mape_pct": 94.15, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "ac76d8899aeb1abe", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 53384.14669731193, "interactivity_tok_s": 19.82880899644883, "output_per_gpu": 844.2204829509132, "total_per_gpu": 1688.4409659018263, "tpot_error_pct": 83.48, "tpot_ms": 50.43167243070884, "tpot_relative": 1.834774, "ttft_error_pct": 94.15, "ttft_ms": 1792.5458007055165, "ttft_relative": 0.058546, "unavailable_metrics": {}}, "concurrency": 2161, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 58736.51590244678, "interactivity_tok_s": 36.38138231292538, "output_per_gpu": 1110.3535610207853, "total_per_gpu": 1480.4714146943804, "tpot_ms": 27.486586171980754, "tpot_relative": 1.0, "ttft_ms": 30617.738248510475, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}]}], "identity": "1024:1024", "label": "1k1k", "precisions": ["fp8"], "rows": 7}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.97, "tpot_shape_error_pct": 3.24, "ttft_mape_pct": 62.97, "ttft_shape_error_pct": 552.57}, "gpu_skus": ["gb200"], "gpus": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 7, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"coverage_pct": 100.0, "points": 7, "status_counts": {"failed": 0, "success": 7, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 16.97, "tpot_shape_error_pct": 3.24, "ttft_mape_pct": 62.97, "ttft_shape_error_pct": 552.57}, "gpu": "gb200", "precisions": ["fp8"], "rows": 7, "topologies": [{"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.94, "tpot_shape_error_pct": null, "ttft_mape_pct": 66.66, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "169d4076c0ccedb4", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 85018.7068752485, "interactivity_tok_s": 32.80708810008704, "output_per_gpu": 270.18037328123125, "total_per_gpu": 2431.623359531082, "tpot_error_pct": 7.94, "tpot_ms": 30.481217868200467, "tpot_relative": 1.079384, "ttft_error_pct": 66.66, "ttft_ms": 53836.420996086366, "ttft_relative": 1.666586, "unavailable_metrics": {}}, "concurrency": 1688, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 61192.381139852405, "interactivity_tok_s": 35.41143610863155, "output_per_gpu": 838.670977701711, "total_per_gpu": 3354.683910806844, "tpot_ms": 28.239464700959974, "tpot_relative": 1.0, "ttft_ms": 32303.40875077036, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 2.99, "tpot_shape_error_pct": null, "ttft_mape_pct": 34.33, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "8180475c3feb1e62", "is_multinode": true, "parallelism": {"attention_dp_size": 32, "moe_ep_size": 32, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 44673.559671216615, "interactivity_tok_s": 38.63195912019555, "output_per_gpu": 202.74597171882593, "total_per_gpu": 1824.7137454694332, "tpot_error_pct": 2.99, "tpot_ms": 25.88530384619381, "tpot_relative": 0.970121, "ttft_error_pct": 34.33, "ttft_ms": 18192.893836560885, "ttft_relative": 0.656695, "unavailable_metrics": {}}, "concurrency": 519, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 54999.9512775252, "interactivity_tok_s": 37.47767258641865, "output_per_gpu": 285.67990469755875, "total_per_gpu": 1469.210938444588, "tpot_ms": 26.682553397469647, "tpot_relative": 1.0, "ttft_ms": 27703.69915191375, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 56}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 7.5, "tpot_shape_error_pct": null, "ttft_mape_pct": 67.69, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "be9fde4ccc44d7dc", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 93443.9635854419, "interactivity_tok_s": 29.681155923822242, "output_per_gpu": 324.0434221784266, "total_per_gpu": 2916.390799605839, "tpot_error_pct": 7.5, "tpot_ms": 33.69141021887881, "tpot_relative": 1.075019, "ttft_error_pct": 67.69, "ttft_ms": 58977.65093153066, "ttft_relative": 1.676858, "unavailable_metrics": {}}, "concurrency": 1484, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 67232.65582641157, "interactivity_tok_s": 31.907803000230597, "output_per_gpu": 1345.0317107165595, "total_per_gpu": 4035.095132149678, "tpot_ms": 31.34029629030783, "tpot_relative": 1.0, "ttft_ms": 35171.53272142667, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 48}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 3, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 3, "status_counts": {"failed": 0, "success": 3, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 18.79, "tpot_shape_error_pct": 3.24, "ttft_mape_pct": 79.72, "ttft_shape_error_pct": 552.57}, "framework": "dynamo-sglang", "id": "1df9348b883ec4c0", "is_multinode": true, "parallelism": {"attention_dp_size": 1, "moe_ep_size": 1, "moe_tp_size": 8, "pp_size": 1, "tp_size": 8}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14463.2235161625, "interactivity_tok_s": 74.91055471540979, "output_per_gpu": 6.93262128227943, "total_per_gpu": 61.66887152824672, "tpot_error_pct": 17.0, "tpot_ms": 13.349253703954897, "tpot_relative": 0.830044, "ttft_error_pct": 71.86, "ttft_ms": 2116.942478349999, "ttft_relative": 0.281418, "unavailable_metrics": {}}, "concurrency": 8, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 22396.274226205423, "interactivity_tok_s": 62.17906325865977, "output_per_gpu": 4.882690263107107, "total_per_gpu": 38.61249868091804, "tpot_ms": 16.082583872968343, "tpot_relative": 1.0, "ttft_ms": 7522.403250460047, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 14444.785081788903, "interactivity_tok_s": 74.67928844462453, "output_per_gpu": 7.772402962189337, "total_per_gpu": 69.50201379239228, "tpot_error_pct": 21.45, "tpot_ms": 13.390593574569344, "tpot_relative": 0.832615, "ttft_error_pct": 66.24, "ttft_ms": 2148.10200181114, "ttft_relative": 0.285561, "unavailable_metrics": {}}, "concurrency": 9, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 16950.221539335325, "interactivity_tok_s": 58.65771905210834, "output_per_gpu": 7.303162217026183, "total_per_gpu": 58.056831887187826, "tpot_ms": 17.048054649238136, "tpot_relative": 1.060032, "ttft_ms": 1292.1659904997796, "ttft_relative": 0.171776, "unavailable_metrics": {}}, "status": "success"}, {"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 22672.138318940055, "interactivity_tok_s": 58.771562363537186, "output_per_gpu": 48.237607357063794, "total_per_gpu": 434.54472637043773, "tpot_error_pct": 17.92, "tpot_ms": 17.015031756590087, "tpot_relative": 1.057979, "ttft_error_pct": 101.05, "ttft_ms": 7006.475802552227, "ttft_relative": 0.931415, "unavailable_metrics": {}}, "concurrency": 90, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 22571.096947365957, "interactivity_tok_s": 48.23819018109982, "output_per_gpu": 54.97247996102101, "total_per_gpu": 440.24439718438276, "tpot_ms": 20.73046265304973, "tpot_relative": 1.289001, "ttft_ms": 3484.961551562883, "ttft_relative": 0.463278, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 3, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}, {"aic": {"coverage_pct": 0.0, "points": 0, "status_counts": {"failed": 1, "success": 0, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": null, "tpot_shape_error_pct": null, "ttft_mape_pct": null, "ttft_shape_error_pct": null}, "aisimulate": {"points": 1, "status_counts": {"failed": 0, "success": 1, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 44.02, "tpot_shape_error_pct": null, "ttft_mape_pct": 33.0, "ttft_shape_error_pct": null}, "framework": "dynamo-sglang", "id": "396ebbb8ed620c94", "is_multinode": true, "parallelism": {"attention_dp_size": 16, "moe_ep_size": 16, "moe_tp_size": 1, "pp_size": 1, "tp_size": 1}, "points": [{"aic": {"e2e_ms": null, "interactivity_tok_s": null, "output_per_gpu": null, "total_per_gpu": null, "tpot_error_pct": null, "tpot_ms": null, "tpot_relative": null, "ttft_error_pct": null, "ttft_ms": null, "ttft_relative": null, "unavailable_metrics": {"e2e_ms": "prediction_failed", "interactivity_tok_s": "prediction_failed", "output_per_gpu": "prediction_failed", "total_per_gpu": "prediction_failed", "tpot_ms": "prediction_failed", "ttft_ms": "prediction_failed"}}, "aic_status": "failed", "aisim_error": null, "aisimulate": {"e2e_ms": 98556.50562749735, "interactivity_tok_s": 19.938618705324263, "output_per_gpu": 375.191526332105, "total_per_gpu": 3376.7237369889453, "tpot_error_pct": 44.02, "tpot_ms": 50.15392564445637, "tpot_relative": 1.440196, "ttft_error_pct": 33.0, "ttft_ms": 47249.03969320846, "ttft_relative": 1.329993, "unavailable_metrics": {}}, "concurrency": 2699, "configuration": {"backend_version": "0.5.14", "forward_model": "op_level"}, "configuration_quality": "estimated", "infx_run_id": "28829786594", "measured": {"e2e_ms": 71151.11133641863, "interactivity_tok_s": 28.715516785961437, "output_per_gpu": 2317.5553339874145, "total_per_gpu": 4635.11066797483, "tpot_ms": 34.824377616246984, "tpot_relative": 1.0, "ttft_ms": 35525.77303499796, "ttft_relative": 1.0, "unavailable_metrics": {}}, "status": "success"}], "precision": "fp8", "rows": 1, "serving": "disaggregated", "spec_method": "none", "total_gpus": 72}]}], "identity": "8192:1024", "label": "8k1k", "precisions": ["fp8"], "rows": 7}]}], "schema_version": 1, "scope": {"claim": "Accuracy applies only to the exact measured operating points in this snapshot; it is not universal model, hardware, or deployment support.", "excluded_multinode_rows": 0, "latency_scope": "client_observed", "measurement_scope": "end_to_end", "metrics": ["TTFT", "TPOT"], "multinode": "included", "published_rows": 2071, "raw_rows": 2071}, "snapshot": {"aic_commit_sha": "c9e7363bf5983b980fe6622f7b064ee7adff5f5d", "aic_source": {"branch": "main", "cli_entry_point": "aisimulate.legacy_cli.entrypoint:main", "commit_sha": "c9e7363bf5983b980fe6622f7b064ee7adff5f5d", "repository": "https://github.com/ai-dynamo/aisimulate"}, "aisimulate_completed_at": "2026-10-19T11:20:04.848919+00:00", "aisimulate_method": "randomized_synthetic_engine_replay", "aisimulate_packages": {"aisimulate": "0.13.0.dev202610170000000074"}, "aisimulate_sot_sha256": "5b751031022d4d5840025f9f96913f255fd3c66eb5d175563123aceded86c02d", "campaign": {"advisory": true, "backend_versions": ["0.24.0", "0.5.14", "1.3.0rc20"], "branch": "main", "cohort_sha256": "e8077a9e31534c300f3371763bde9e6be15d47307247189c67890990a60c8b46", "commit_sha": "c9e7363bf5983b980fe6622f7b064ee7adff5f5d", "completed_at": "2026-10-19T11:20:04.848919+00:00", "configuration": {"counts": {"estimated": 2031, "verified": 40}, "profile": "coverage-experiment/1"}, "dataset_sha256": "a9b38d3cadcd5a0ce7f8ff4f2269f5085e8007e29311099f570205dd09526e3b", "driver_sha256": "53646fd331dbb85cc7784bd0056e6af3946686b984ed865d1542e8413e715e79", "exclusion_reasons": {"source_unresolved": 210}, "measurement_filter_counts": {"source_filter": 35895, "stale": 1012, "superseded_image": 697, "superseded_row": 45424}, "measurement_sha256": "9125257cb98e195918fee2ee451d9ee4057fed5fab40e0da43cfc054d4a2330a", "metric_contract": "serving-metrics-v1", "outcomes": {"evaluated": 2071, "unsupported": 210}, "published": 2071, "release_tag": "db-dump/2026-09-28", "run_attempt": "1", "run_id": "39237616407", "schema_version": 1, "selected": 2281, "selection_policy": "gym-resolved-config-v2", "started_at": "2026-10-19T10:50:33.435158+00:00", "status": "complete", "wheel_sha256": "379472f58a50f78607140e8a8333d3e65ab2e51555c9d2c5df5e5070f5f3b74a"}, "corrections": [], "evaluated_revision": {"branch": "main", "commit_sha": "c9e7363bf5983b980fe6622f7b064ee7adff5f5d"}, "measurement_date_through": "2026-09-27", "measurement_source": "SemiAnalysis InferenceX", "measurement_source_url": "https://github.com/SemiAnalysisAI/InferenceX-app/releases/tag/db-dump/2026-09-28", "prediction_generated_at": "2026-10-19T10:50:33.435158+00:00", "predictions_sha256": "cb2c66ce86cc1a35e09dfd8032432d16be0b49c727198392cc1fa7adf287e2ba", "release_tag": "db-dump/2026-09-28"}, "title": "AISimulate E2E Accuracy Overview", "totals": {"aic": {"coverage_pct": 62.68, "points": 1298, "status_counts": {"failed": 773, "success": 1298, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.25, "tpot_shape_error_pct": 12.05, "ttft_mape_pct": 48.02, "ttft_shape_error_pct": 38.41}, "aisimulate": {"coverage_pct": 76.48, "points": 1584, "status_counts": {"failed": 487, "success": 1584, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 22.39, "tpot_shape_error_pct": 11.83, "ttft_mape_pct": 45.27, "ttft_shape_error_pct": 33.2}, "by_configuration_quality": {"estimated": {"aic": {"coverage_pct": 63.12, "points": 1282, "status_counts": {"failed": 749, "success": 1282, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 21.6, "tpot_shape_error_pct": 12.06, "ttft_mape_pct": 47.72, "ttft_shape_error_pct": 38.17}, "aisimulate": {"points": 1552, "tpot_mape_pct": 21.66, "tpot_shape_error_pct": 11.89, "ttft_mape_pct": 45.29, "ttft_shape_error_pct": 33.34}, "rows": 2031}, "verified": {"aic": {"coverage_pct": 40.0, "points": 16, "status_counts": {"failed": 24, "success": 16, "unknown": 0, "unsupported": 0}, "tpot_mape_pct": 74.7, "tpot_shape_error_pct": 10.83, "ttft_mape_pct": 72.05, "ttft_shape_error_pct": 62.49}, "aisimulate": {"points": 32, "tpot_mape_pct": 57.6, "tpot_shape_error_pct": 8.46, "ttft_mape_pct": 43.97, "ttft_shape_error_pct": 25.53}, "rows": 40}}, "frameworks": ["dynamo-sglang", "dynamo-trt", "dynamo-vllm", "sglang", "trt", "vllm"], "gpu_skus": ["b200", "b300", "gb200", "gb300", "h100", "h200"], "models": 21, "precisions": ["bf16", "fp4", "fp8", "int4"], "rows": 2071, "workloads": ["1024:1024", "1024:8192", "8192:1024"]}}