{ "model": "glm-5.2", "endpoint": "http://:8211", "config": { "max_model_len": 262144, "mtp": true, "async_scheduling": true, "enforce_eager": false, "kv_cache_dtype": "auto", "decode_context_parallel_size": 2, "attention_backend": "B12X_MLA_SPARSE", "num_speculative_tokens": 4 }, "concurrency": [ { "n": 1, "wall_s": 55.46264585899189, "agg_prefill_tps": 624.8844399409352, "agg_decode_tps": 26.016969438265658, "total_in": 10086, "total_out": 1024, "per": [ { "ttft_s": 16.14058433100581, "duration_s": 55.461074844992254, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 624.8844399409352, "decode_tps": 26.016969438265658 } ] }, { "n": 2, "wall_s": 83.99196182697779, "agg_prefill_tps": 488.9274769412379, "agg_decode_tps": 24.628340059622264, "total_in": 20172, "total_out": 2048, "per": [ { "ttft_s": 41.25714911596151, "duration_s": 83.98940804396989, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 244.4667219165161, "decode_tps": 23.939759461896507 }, { "ttft_s": 0.9148851740173995, "duration_s": 40.3648982369923, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11024.334295101586, "decode_tps": 25.931550348715046 } ] }, { "n": 4, "wall_s": 163.0478340940317, "agg_prefill_tps": 327.3571147984901, "agg_decode_tps": 25.211301927781758, "total_in": 40344, "total_out": 4096, "per": [ { "ttft_s": 0.7360965470434166, "duration_s": 42.4066287750029, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 13702.00694529967, "decode_tps": 24.549722437036753 }, { "ttft_s": 123.24017099099001, "duration_s": 163.0433470539865, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 81.8401980368672, "decode_tps": 25.70146659605499 }, { "ttft_s": 83.63244688004488, "duration_s": 122.36600818502484, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 120.59912601225793, "decode_tps": 26.411204276960543 }, { "ttft_s": 43.28443912300281, "duration_s": 82.75020404998213, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 233.0167654786581, "decode_tps": 25.921200359166576 } ] }, { "n": 8, "wall_s": 347.3067554680165, "agg_prefill_tps": 261.51172584566035, "agg_decode_tps": 23.61453365923059, "total_in": 80688, "total_out": 8192, "per": [ { "ttft_s": 0.7353129350231029, "duration_s": 39.86021272104699, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 13716.608969598918, "decode_tps": 26.147031828703465 }, { "ttft_s": 215.17515375901712, "duration_s": 268.0456537880236, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 46.87344158376063, "decode_tps": 19.349164457282384 }, { "ttft_s": 129.8861644230201, "duration_s": 173.58394229301484, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 77.65261253809439, "decode_tps": 23.410801415200723 }, { "ttft_s": 94.06110376602737, "duration_s": 129.01135029899888, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 107.228169734096, "decode_tps": 29.270179797871187 }, { "ttft_s": 268.99189491500147, "duration_s": 307.6762964410009, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 37.495553548879485, "decode_tps": 26.44476739060966 }, { "ttft_s": 40.707678512029815, "duration_s": 93.0779599819798, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 247.76652387631032, "decode_tps": 19.5339794113384 }, { "ttft_s": 174.44189829594688, "duration_s": 214.43541216297308, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 57.81867830220892, "decode_tps": 25.57914774384058 }, { "ttft_s": 308.54448204598157, "duration_s": 347.30204227595823, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 32.688965730707544, "decode_tps": 26.394850293202165 } ] }, { "n": 16, "wall_s": 635.6851423639455, "agg_prefill_tps": 271.8680892273065, "agg_decode_tps": 25.78594764615557, "total_in": 161376, "total_out": 16384, "per": [ { "ttft_s": 39.10956927598454, "duration_s": 75.76292640098836, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 257.8908483707942, "decode_tps": 27.910131028683864 }, { "ttft_s": 118.04273068998009, "duration_s": 160.29913651500829, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 85.44363503830853, "decode_tps": 24.209347199001098 }, { "ttft_s": 161.17016053397674, "duration_s": 204.66098471498117, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 62.57982226104281, "decode_tps": 23.522203114440348 }, { "ttft_s": 205.5366172940121, "duration_s": 243.2379070490133, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 49.07154809097773, "decode_tps": 27.13435022111666 }, { "ttft_s": 278.49332659499487, "duration_s": 314.2989224420162, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 36.216307669977994, "decode_tps": 28.570953109417495 }, { "ttft_s": 513.5210573450313, "duration_s": 552.3262221920304, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 19.64086935820294, "decode_tps": 26.362470151421377 }, { "ttft_s": 352.7563637899584, "duration_s": 393.0306978339795, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 28.591971783691204, "decode_tps": 25.400792447165717 }, { "ttft_s": 593.5729412430082, "duration_s": 635.6681349629653, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 16.992014458878106, "decode_tps": 24.302061817451694 }, { "ttft_s": 475.8419849119964, "duration_s": 512.7739716849755, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 21.196111986346338, "decode_tps": 27.69956586111598 }, { "ttft_s": 315.0187293199706, "duration_s": 351.7881105369888, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 32.01714393862422, "decode_tps": 27.822061893348305 }, { "ttft_s": 553.1502134540351, "duration_s": 592.69952990202, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 18.233745110609295, "decode_tps": 25.866439470463305 }, { "ttft_s": 431.9172281139763, "duration_s": 474.9734737050021, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 23.35169644434387, "decode_tps": 23.759619213367355 }, { "ttft_s": 393.9135866590077, "duration_s": 431.0215094339801, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 25.604600454492502, "decode_tps": 27.568236740267416 }, { "ttft_s": 76.49644242000068, "duration_s": 117.14609883300727, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 131.84926881466222, "decode_tps": 25.16626437395108 }, { "ttft_s": 244.10936195298564, "duration_s": 277.80721757595893, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 41.3175468540306, "decode_tps": 30.358014807997943 }, { "ttft_s": 0.9128400010522455, "duration_s": 38.06923376501072, "in_tokens": 10086, "out_tokens": 1024, "prefill_tps": 11049.033771935612, "decode_tps": 27.532273624258583 } ] } ], "niah": { "in_tokens": 240074, "ttft_s": 383.7619993999833, "duration_s": 384.59631163900485, "out_tokens": 25, "prefill_tps": 625.5804388536611, "decode_tps": 28.766208713594942, "found": true, "needle_value": "BANANA-MOOSE-7421", "answer_head": " If the text does not contain the code, reply with \"NOT FOUND\".\n\nBANANA-MOOSE-7421" } }