{ "model": "glm-5.2", "endpoint": "http://:8210", "config": { "max_model_len": 128000, "mtp": false, "async_scheduling": false, "enforce_eager": true, "kv_cache_dtype": "fp8_ds_mla" }, "concurrency": [ { "n": 1, "wall_s": 83.46870014700107, "agg_prefill_tps": 23730.91664685037, "agg_decode_tps": 12.319039106857042, "total_in": 10086, "total_out": 1024, "per": [ { "ttft_s": 0.4250151879969053, "duration_s": 83.46720631397329, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 23730.91664685037, "decode_tps": 12.319039106857042 } ] }, { "n": 2, "wall_s": 87.5483156919945, "agg_prefill_tps": 23861.97882361139, "agg_decode_tps": 23.45067654874787, "total_in": 20172, "total_out": 2048, "per": [ { "ttft_s": 0.8448660710128024, "duration_s": 87.54601570201339, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11937.9867958352, "decode_tps": 11.799151503225504 }, { "ttft_s": 0.29956138995476067, "duration_s": 87.36476089898497, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 33669.225535117104, "decode_tps": 11.749815147370066 } ] }, { "n": 4, "wall_s": 105.01616257702699, "agg_prefill_tps": 43985.403645281614, "agg_decode_tps": 39.081311826937394, "total_in": 40344, "total_out": 4096, "per": [ { "ttft_s": 0.9164160549989901, "duration_s": 105.01215200102888, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11005.9180488835, "decode_tps": 9.827491882380185 }, { "ttft_s": 0.30762922001304105, "duration_s": 104.68880904599791, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 32786.22232170414, "decode_tps": 9.800617330686007 }, { "ttft_s": 0.915373956959229, "duration_s": 105.011793638987, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11018.44762276674, "decode_tps": 9.827427332513922 }, { "ttft_s": 0.9172133629908785, "duration_s": 105.01317776099313, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 10996.350911320404, "decode_tps": 9.827470314686213 } ] }, { "n": 8, "wall_s": 211.22031849296764, "agg_prefill_tps": 667.7085918195853, "agg_decode_tps": 38.802276633220906, "total_in": 80688, "total_out": 8192, "per": [ { "ttft_s": 1.0292239459813572, "duration_s": 120.49851802602643, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 9799.616535722043, "decode_tps": 8.562869713740708 }, { "ttft_s": 1.0304710670025088, "duration_s": 120.49879872903693, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 9787.756612457557, "decode_tps": 8.562938981568225 }, { "ttft_s": 1.0277425730018876, "duration_s": 120.49542883899994, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 9813.74155839458, "decode_tps": 8.562984954126112 }, { "ttft_s": 1.0274803399806842, "duration_s": 120.49637578701368, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 9816.246216632824, "decode_tps": 8.56289828555041 }, { "ttft_s": 120.84070815495215, "duration_s": 211.21325369097758, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 83.46525069239813, "decode_tps": 11.319809505556076 }, { "ttft_s": 0.298364263959229, "duration_s": 119.94888777396409, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 33804.31646257152, "decode_tps": 8.549899908415021 }, { "ttft_s": 1.0294912179815583, "duration_s": 120.49788634100696, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 9797.072402205451, "decode_tps": 8.562934146278115 }, { "ttft_s": 120.49913692096015, "duration_s": 211.05247776099714, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 83.70184432620277, "decode_tps": 11.297208810960775 } ] }, { "n": 16, "wall_s": 347.18223452800885, "agg_prefill_tps": 649.7799636587389, "agg_decode_tps": 47.188063449126105, "total_in": 161376, "total_out": 16384, "per": [ { "ttft_s": 122.9887021080358, "duration_s": 247.88026835903293, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.00753261986821, "decode_tps": 8.19110553825593 }, { "ttft_s": 122.36689803999616, "duration_s": 247.4718603939982, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.42425166897135, "decode_tps": 8.177133670407716 }, { "ttft_s": 247.88314656500006, "duration_s": 347.00379909202456, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 40.688526589100896, "decode_tps": 10.320755300930719 }, { "ttft_s": 122.98994975897949, "duration_s": 247.8806138219661, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.00670070819035, "decode_tps": 8.191164709349822 }, { "ttft_s": 122.98767868900904, "duration_s": 247.87981558503816, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.00821503025367, "decode_tps": 8.191068112251395 }, { "ttft_s": 248.3539364010212, "duration_s": 347.174491420039, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 40.61139576106404, "decode_tps": 10.352097291936138 }, { "ttft_s": 0.3048746809945442, "duration_s": 121.96259336097864, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 33082.445439870724, "decode_tps": 8.408837606851414 }, { "ttft_s": 0.9096906300401315, "duration_s": 122.36533842602512, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11087.285794682804, "decode_tps": 8.422827744646204 }, { "ttft_s": 122.99076496699126, "duration_s": 247.88210159598384, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.00615715096104, "decode_tps": 8.19112059821224 }, { "ttft_s": 0.9164128599804826, "duration_s": 122.37234718800755, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11005.956420357095, "decode_tps": 8.422807873982435 }, { "ttft_s": 122.98619927599793, "duration_s": 247.8764023249969, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 82.00920151508731, "decode_tps": 8.191194945840868 }, { "ttft_s": 0.9112317039980553, "duration_s": 122.36756711598719, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11068.534990329446, "decode_tps": 8.422780059433755 }, { "ttft_s": 248.34920644800877, "duration_s": 347.16969710600097, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 40.612169228378335, "decode_tps": 10.35210403417749 }, { "ttft_s": 0.9128333679982461, "duration_s": 122.36809424101375, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11049.114059138315, "decode_tps": 8.422854577452778 }, { "ttft_s": 0.909883382031694, "duration_s": 122.3648867320153, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11084.937036083458, "decode_tps": 8.422872436569227 }, { "ttft_s": 248.3545818660059, "duration_s": 347.175478707999, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 40.61129021344842, "decode_tps": 10.35206148387519 } ] } ], "niah": { "in_tokens": 126579, "ttft_s": 232.2551459139795, "duration_s": 232.2551459139795, "out_tokens": 1, "prefill_tps": 544.9997652447329, "decode_tps": 0, "found": false, "needle_value": "BANANA-MOOSE-7421", "answer_head": "" } }