framework,version,device,op_name,bmm_dtype,num_tokens,num_heads,latency
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,128,0.006864000111818314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,128,0.016012799739837647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,8,0.014934399724006652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,4,0.017411200702190398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,4,0.007014399766921997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,2,0.014441600441932679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,64,0.004985599964857102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,1,0.00923520028591156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,8,0.007372800260782242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,16,0.011561600118875503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1,32,0.014742399752140044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,64,0.010921599715948105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,32,0.008736000210046769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,2,0.006473600119352341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,16,0.007187200337648391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,128,0.006239999830722809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,64,0.005100800096988678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1,1,0.012697599828243256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,64,0.013955199718475341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,128,0.016492800414562227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,32,0.008575999736785888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,32,0.00459199994802475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,8,0.004095999896526337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,16,0.004361600056290627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,2,0.004188799858093261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,4,0.004134399816393852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,16,0.0072672002017498015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2,1,0.004112000018358231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,8,0.007036799937486649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,4,0.0067103996872901915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,128,0.00626240000128746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,64,0.004995200037956238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,2,0.006204799935221672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,32,0.004556800052523613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2,1,0.005740800127387047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,128,0.01669439971446991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,16,0.004262400045990944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,16,0.007651200145483017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,8,0.004108799993991852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,64,0.01138240024447441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,8,0.0070592001080513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,4,0.004262400045990944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,32,0.008723200112581254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,2,0.0041184000670909885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4,1,0.0044895999133586885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,128,0.006627199798822403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,64,0.005267199873924255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,4,0.006691200286149978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,2,0.006508799642324448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4,1,0.00605119988322258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,128,0.01685120016336441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,32,0.004524800181388855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,16,0.004348799958825112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,64,0.0114656001329422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,8,0.004303999990224838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,4,0.004124800115823746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,32,0.008803199976682663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,2,0.0043136000633239744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,16,0.007692799717187881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,2,0.006505600363016129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,8,0.007215999811887741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8,1,0.00419199988245964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,1,0.00618240013718605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8,4,0.007260800153017044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,128,0.018016000092029572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,128,0.006297600269317627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,64,0.0050432000309228895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,64,0.011817599833011627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,16,0.00445760004222393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,32,0.004595199972391129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,8,0.004380799829959869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,4,0.0040224000811576845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,8,0.0073632001876831055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,2,0.004483199864625931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,32,0.009276799857616425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,4,0.006764800101518631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,16,0.009382399916648864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,16,1,0.004291199892759323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,128,0.006716799736022949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,128,0.01993599981069565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,32,0.005161599814891815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,2,0.006934399902820587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,64,0.005459199845790863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,64,0.012556800246238708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,16,1,0.006460800021886826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,32,0.009398400038480758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,8,0.004838399961590767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,16,0.008297599852085114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,16,0.0045855998992919925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,8,0.00759039968252182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,4,0.004307200014591217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,4,0.007254400104284286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,2,0.004649600014090538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,32,1,0.005676800012588501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,1,0.006790400296449661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,128,0.007436800003051758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,64,0.013273599743843078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,64,0.005728000029921532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,32,0.00490880012512207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,32,2,0.007011199742555619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,16,0.004675199836492538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,32,0.00984319970011711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,8,0.0042975999414920805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,128,0.02144639939069748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,4,0.004150399938225746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,2,0.007599999755620956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,4,0.007635200023651123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,2,0.004412800073623657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,16,0.00854720026254654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,48,1,0.0040608000010252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,1,0.007078400254249573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,48,8,0.007740800082683563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,128,0.007900799810886382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,64,0.006172800064086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,32,0.005110400170087815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,32,0.009881599992513656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,16,0.008959999680519104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,16,0.004652800038456917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,8,0.004239999875426293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,64,0.014716799557209014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,128,0.02298559993505478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,4,0.004323200136423111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,4,0.007807999849319458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,2,0.007727999985218048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,2,0.00493439994752407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,64,1,0.004047999903559685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,8,0.008032000064849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,64,1,0.007382400333881378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,64,0.00639680027961731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,32,0.005263999849557876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,128,0.008582399785518646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,16,0.0047231998294591905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,8,0.004335999861359597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,4,0.005107200145721436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,128,0.02497279942035675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,2,0.004124800115823746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,64,0.015836800634860992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,32,0.01061440035700798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,16,0.008995199948549271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,80,1,0.004227200150489807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,4,0.0076672002673149105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,128,0.00904960036277771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,64,0.00663359984755516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,8,0.007971200346946716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,32,0.005296000093221664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,128,0.025868800282478333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,2,0.0074143998324871065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,80,1,0.006857600063085556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,16,0.00469760000705719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,64,0.016332800686359405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,4,0.0053311999887228016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,8,0.0043136000633239744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,2,0.004291199892759323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,32,0.010831999778747558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,96,1,0.005305600166320801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,8,0.008131200075149536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,128,0.015174399316310882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,4,0.007817599922418594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,64,0.007820799946784973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,16,0.00941760018467903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,2,0.008054400235414505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,32,0.005836800113320351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,96,1,0.007257600128650665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,16,0.005324799939990044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,16,0.00958079993724823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,128,0.02892799973487854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,8,0.004371200129389763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,4,0.004291199892759323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,64,0.016764800250530242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,2,0.005014400184154511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,32,0.011721599847078323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,128,1,0.0042559999972581865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,1,0.007673600316047668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,8,0.008508799970149994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,128,0.016208000481128693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,4,0.008111999928951263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,64,0.008364800363779068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,64,0.022937600314617158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,128,2,0.007884799689054488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,32,0.00628800019621849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,128,0.04279359877109527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,16,0.004892800003290176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,8,0.004428799822926521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,2,0.004912000149488449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,4,0.004342399910092354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,32,0.014672000706195832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,160,1,0.004121600091457367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,16,0.010595200210809707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,128,0.020902399718761445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,2,0.008012799918651581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,64,0.008703999966382981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,8,0.00910400003194809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,4,0.008032000064849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,160,1,0.007744000107049942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,128,0.04627839922904968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,32,0.015203200280666351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,16,0.0049183998256921765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,32,0.006662400066852569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,8,0.004790399968624115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,2,0.00421760007739067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,64,0.02401600033044815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,4,0.0042975999414920805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,16,0.010371199995279311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,2,0.00782719999551773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,192,1,0.004470400139689445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,8,0.009276799857616425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,128,0.025625601410865784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,1,0.007599999755620956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,128,0.0516543984413147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,192,4,0.00820159986615181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,64,0.010003200173377991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,64,0.02627519965171814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,32,0.007340800017118454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,16,0.0058143999427556995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,8,0.00480320006608963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,4,0.004412800073623657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,2,0.008102399855852127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,2,0.004332799836993218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,32,0.016233600676059723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,16,0.011001600325107575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,256,1,0.004265600070357323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,128,0.029875200986862183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,8,0.009161599725484849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,4,0.00823040008544922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,64,0.012800000607967377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,32,0.007680000364780426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,16,0.0059519998729228975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,256,1,0.007587199658155441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,128,0.067740797996521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,8,0.005203200131654739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,4,0.004572800174355507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,64,0.03636159896850586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,32,0.020329600572586058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,2,0.004470400139689445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,128,0.034108799695968625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,320,1,0.004204799979925155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,1,0.008630400151014328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,16,0.01297920048236847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,64,0.01607999950647354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,128,0.0704479992389679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,8,0.009241600334644318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,4,0.008640000224113464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,64,0.037945601344108584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,32,0.0086496002972126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,16,0.006518399715423584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,32,0.020473599433898926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,320,2,0.00814720019698143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,8,0.005132799968123436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,4,0.00453759990632534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,2,0.0052767999470233916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,384,1,0.004771199822425842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,16,0.013462400436401368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,128,0.04169279932975769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,8,0.00912640020251274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,64,0.02298559993505478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,4,0.008483199775218964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,2,0.007983999699354172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,32,0.010134399682283402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,384,1,0.0077632002532482145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,16,0.007478400319814682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,16,0.015488000214099884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,128,0.09018880128860474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,8,0.005555199831724167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,64,0.04805119931697845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,4,0.004873599857091904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,32,0.025750398635864258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,2,0.004342399910092354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,512,1,0.004355200007557869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,1,0.007568000257015229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,128,0.08752959966659546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,128,0.13018560409545898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,64,0.03167040050029755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,64,0.06837120056152343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,4,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,8,0.011001600325107575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,32,0.014585599303245544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,32,0.03745599985122681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,512,2,0.00833280012011528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,16,0.00878399983048439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,16,0.020108799636363982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,8,0.006492800265550614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,4,0.0050976000726222995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,2,0.004569600149989128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,768,1,0.004425600171089172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,8,0.013507199287414551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,128,0.07343680262565613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,128,0.16862720251083374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,64,0.05842559933662415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,64,0.08831359744071961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,32,0.021804800629615782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,32,0.04718399941921234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,4,0.009193599969148637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,2,0.008745600283145905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,768,1,0.007804799824953079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,16,0.009932799637317658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,8,0.007497599720954895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,2,0.005289600044488907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,4,0.007011199742555619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1024,1,0.004361600056290627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,16,0.02503040134906769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,128,0.10592960119247437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,8,0.015539200603961944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,128,0.2452608108520508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,64,0.055955201387405396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,32,0.03027839958667755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,16,0.014844800531864166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,4,0.010921599715948105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,32,0.06733760237693787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,2,0.008991999924182892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1024,1,0.008528000116348267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,8,0.008668799698352814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,64,0.12857279777526856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,1,0.005263999849557876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,4,0.006351999938488007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,1536,2,0.004937599971890449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,16,0.03639360070228577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,128,0.1381376028060913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,64,0.07158079743385315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,128,0.3244607925415039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,4,0.013414399325847625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,8,0.01995840072631836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,64,0.1668544054031372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,32,0.03920960128307342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,16,0.02038400024175644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,4,0.007449600100517273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,2,0.009222400188446046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,8,0.00997759997844696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,32,0.08671039938926697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,1536,1,0.00843520015478134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,8,0.024953599274158477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,2,0.005670399963855743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,4,0.01518079936504364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,16,0.04636479914188385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,2048,1,0.004700800031423568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,128,0.30899839401245116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,128,0.4776447772979736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,64,0.10492160320281982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,32,0.05575360059738159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,32,0.1257472038269043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,64,0.24167039394378662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,16,0.043375998735427856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,16,0.06629760265350342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,8,0.014672000706195832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,2,0.010982400178909302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,2048,1,0.009379199892282485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,8,0.035657599568367004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,4,0.00868159979581833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,4,0.019702400267124175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,2,0.00650240033864975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,3072,1,0.005340800061821938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,128,0.27287681102752687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,128,0.6270400047302246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,64,0.13622080087661742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,64,0.3194272041320801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,32,0.16402560472488403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,32,0.1060256004333496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,2,0.012905600666999816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,4,0.010342399775981902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,16,0.04046719968318939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,16,0.0862496018409729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,8,0.01966720074415207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,2,0.007193599641323089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,3072,1,0.009164799749851228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,4096,1,0.005449600145220756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,8,0.045731198787689206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,4,0.02454719990491867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,128,0.43073601722717286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,128,0.9292096138000489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,64,0.20316801071166993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,2,0.015238399803638458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,64,0.4695903778076172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,16,0.05877760052680969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,32,0.10280640125274658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,32,0.24080960750579833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,16,0.12508480548858641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,8,0.029654398560523987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,4096,1,0.011222399771213531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,4,0.014528000354766845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,2,0.008575999736785888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,128,0.5617663860321045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,6144,1,0.006345599889755249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,1,0.013046400249004364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,8,0.0655456006526947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,64,0.2801471948623657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,2,0.019801600277423857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,6144,4,0.034985598921775815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,32,0.13556159734725953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,16,0.07063040137290955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,32,0.31818881034851076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,128,1.2097599983215332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,64,0.6222239971160889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,8,0.05549439787864685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,8,0.08498560190200806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,4,0.019760000705718993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,2,0.01016639992594719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,float16,8192,1,0.007664000242948532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,16,0.16296000480651857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,4,0.04572800099849701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,2,0.02436479926109314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_pre,fp8,8192,1,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,128,0.015187199413776397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,16,0.014537599682807923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,64,0.014732800424098969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,128,0.013126400113105775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,16,0.01512639969587326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,64,0.01629119962453842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,2,0.014476799964904785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,8,0.007129599899053573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,32,0.008329600095748901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,1,0.013510400056838989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,32,0.004787199944257736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,4,0.006752000004053116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,2,0.007228799909353256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,64,0.0050016000866889955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,64,0.009567999839782714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,128,0.013219200074672699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1,1,0.014732800424098969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,128,0.01225920021533966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,32,0.005107200145721436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,16,0.008329600095748901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,32,0.00820479989051819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,8,0.014688000082969666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,16,0.004691199958324432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,4,0.004806400090456009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,8,0.004495999962091446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,2,0.004479999840259552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1,4,0.00461760014295578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2,1,0.004896000027656555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,8,0.007148800045251846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,4,0.0068063996732234955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,2,0.006886400282382965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,128,0.005990400165319443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,64,0.009900800138711929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,64,0.005129599943757057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,8,0.004524800181388855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,16,0.004700800031423568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,32,0.004870399832725525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2,1,0.006454399973154068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,128,0.012294399738311767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,4,0.004704000055789947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,32,0.008284799754619598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,2,0.006803199648857117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,2,0.004355200007557869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4,1,0.004374400153756142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,16,0.007814399898052216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,8,0.007187200337648391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,128,0.006172800064086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,4,0.0070271998643875126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4,1,0.0067552000284194945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,64,0.004873599857091904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,8,0.004694399982690811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,16,0.004620800167322159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,32,0.004838399961590767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,128,0.013411200046539307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,64,0.010131199657917023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,4,0.004646399989724159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,8,0.007513599842786789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,4,0.007235199958086014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,2,0.004652800038456917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,1,0.006742399930953979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,32,0.008556800335645676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,16,0.007766400277614593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8,2,0.007238399982452392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8,1,0.004390399903059006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,64,0.011449600011110306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,128,0.014268800616264343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,128,0.0062015999108552934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,64,0.005040000006556511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,32,0.004787199944257736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,8,0.004630399867892265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,16,0.005056000128388405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,4,0.0045311998575925825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,2,0.005027199909090996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,32,0.008819200098514557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,16,0.008099199831485748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16,1,0.004985599964857102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,8,0.007344000041484833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,2,0.007180800288915634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,4,0.0074143998324871065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,128,0.006822399795055389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,64,0.005171199887990951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,64,0.012399999797344208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,32,0.004687999933958053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,16,0.004569600149989128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16,1,0.0068960003554821014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,32,0.010307200253009796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,8,0.0046847999095916745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,128,0.017123199999332428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,4,0.004550400003790855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,4,0.007574400305747986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,2,0.004527999833226204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,2,0.007526399940252304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,16,0.008479999750852585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,32,1,0.004508800059556961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,8,0.008163200318813324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,32,1,0.0072512000799179075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,64,0.01385599970817566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,128,0.006937599927186966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,64,0.005455999821424484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,8,0.004707200080156326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,32,0.0048191998153924945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,32,0.010902400314807891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,16,0.004899200052022934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,16,0.008873599767684936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,8,0.008297599852085114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,4,0.007919999957084655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,128,0.019888000190258028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,4,0.004675199836492538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,128,0.022892799973487855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,2,0.004527999833226204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,48,1,0.009065599739551544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,1,0.007494399696588517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,128,0.007238399982452392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,64,0.005619199946522713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,32,0.004921599850058555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,4,0.004662400111556053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,32,0.011289600282907486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,48,2,0.007900799810886382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,8,0.0045855998992919925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,16,0.005056000128388405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,16,0.00995199978351593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,4,0.008246400207281113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,64,0.015494400262832641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,2,0.004499199986457825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,2,0.007804799824953079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,64,1,0.004412800073623657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,1,0.008006399869918824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,128,0.007612799853086471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,64,8,0.008489599823951722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,64,0.005878400057554245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,32,0.005142400041222572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,16,0.0049472000449895855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,8,0.004681599885225296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,4,0.005075199902057648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,2,0.004633599892258644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,128,0.027427199482917785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,32,0.01218239963054657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,64,0.017360000312328337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,16,0.010467199981212616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,80,1,0.004518400132656098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,128,0.008556800335645676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,8,0.008819200098514557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,4,0.008287999778985977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,64,0.005872000008821487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,32,0.004995200037956238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,1,0.007612799853086471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,80,2,0.008003199845552445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,16,0.004956800118088722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,128,0.03028480112552643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,16,0.010867200046777725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,64,0.018688000738620758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,8,0.004623999819159508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,4,0.008588799834251403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,4,0.00456320010125637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,32,0.012809599936008453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,2,0.0046431999653577805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,8,0.00899839997291565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,96,1,0.004495999962091446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,128,0.009670399874448777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,64,0.02136960029602051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,64,0.006393600255250931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,32,0.005340800061821938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,2,0.008217599987983704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,8,0.010236799716949463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,96,1,0.0077632002532482145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,16,0.0049727998673915865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,128,0.038796800374984744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,8,0.004921599850058555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,4,0.0047231998294591905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,32,0.014499199390411378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,2,0.004611200094223023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,16,0.011423999816179276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,128,1,0.004575999826192856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,4,0.008886399865150451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,128,0.017478400468826295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,64,0.0067391999065876005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,2,0.008540800213813782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,32,0.0055808000266551975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,128,1,0.008179199695587159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,128,0.04738239943981171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,16,0.0050976000726222995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,8,0.005206400156021118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,4,0.004783999919891357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,2,0.004732799902558327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,32,0.01693760007619858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,2,0.00873280018568039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,8,0.010732799768447876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,160,1,0.004499199986457825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,64,0.025062400102615356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,128,0.02082560062408447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,16,0.012406399846076966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,4,0.009267199784517288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,32,0.01794559955596924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,64,0.007280000299215317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,32,0.005804799869656563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,16,0.004953600093722343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,8,0.005110400170087815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,160,1,0.008019199967384339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,128,0.05211840271949768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,8,0.01122559979557991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,2,0.0047391999512910845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,4,0.004675199836492538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,64,0.028035199642181395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,16,0.012889599800109864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,2,0.008790399879217148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,128,0.02423679977655411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,192,1,0.004998400062322617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,128,0.06468480229377746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,32,0.02048960030078888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,64,0.008083199709653854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,4,0.009068799763917923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,32,0.006032000109553337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,8,0.004956800118088722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,16,0.005260799825191498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,192,1,0.008246400207281113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,2,0.004764800146222115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,4,0.005011200159788132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,8,0.01125440001487732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,64,0.034534400701522826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,2,0.008953599631786347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,1,0.008211199939250947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,256,1,0.00469760000705719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,16,0.014185599982738495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,128,0.02778559923171997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,32,0.007116799801588058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,256,4,0.010022400319576264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,64,0.008947200328111648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,16,0.005369599908590317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,8,0.005401600152254105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,128,0.08055359721183777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,4,0.0048767998814582825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,2,0.004646399989724159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,64,0.04577600061893463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,1,0.008403199911117553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,320,1,0.004630399867892265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,128,0.09228799939155578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,128,0.03213759958744049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,16,0.016143999993801117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,4,0.010441599786281586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,8,0.01199359968304634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,32,0.02423039972782135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,320,2,0.009014400094747544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,64,0.012908799946308136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,32,0.007580800354480744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,16,0.005382400006055832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,64,0.050387197732925416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,4,0.004796800017356872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,8,0.005187200009822845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,4,0.011017599701881408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,2,0.004543999955058098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,32,0.026707199215888978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,16,0.017420800030231477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,384,1,0.004604800045490265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,8,0.01268800050020218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,1,0.008591999858617782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,128,0.039427199959754945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,384,2,0.008963199704885483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,128,0.11885759830474854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,64,0.021807999908924104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,32,0.007952000200748443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,8,0.005340800061821938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,16,0.006019200012087822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,2,0.0047807998955249785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,4,0.0049632001668214795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,64,0.06250240206718445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,4,0.011007999628782272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,32,0.0330592006444931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,512,1,0.004854400083422661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,2,0.010054399818181991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,16,0.020684799551963805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,8,0.013900800049304963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,512,1,0.008364800363779068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,64,0.09017279744148254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,128,0.05407040119171143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,128,0.17394880056381226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,64,0.02924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,8,0.005481600016355515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,32,0.009974399954080582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,16,0.0075103998184204105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,4,0.005222399905323982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,2,0.005078399926424027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,2,0.0107744000852108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,32,0.04938879907131195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,128,0.06928319931030273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,768,1,0.004950400069355964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,64,0.11718080043792725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,128,0.2445215940475464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,64,0.03656960129737854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,32,0.01998399943113327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,8,0.017212800681591034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,16,0.02646079957485199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,1,0.008950400352478027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,768,4,0.012220799922943115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,16,0.008131200075149536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,8,0.005836800113320351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,32,0.061615997552871705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,8,0.02025599926710129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,4,0.005008000135421753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,2,0.005011200159788132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,2,0.011088000237941742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,16,0.03236159980297089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1024,1,0.00461760014295578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,128,0.09793279767036438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,4,0.013868799805641175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,64,0.05295360088348389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,32,0.027667200565338133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,16,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,128,0.333734393119812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1024,1,0.010476800054311753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,8,0.007395199686288834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,64,0.18220800161361694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,2,0.005187200009822845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,32,0.08863040208816528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,4,0.005283199995756149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,16,0.0487744003534317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,1,0.011212799698114395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,1536,1,0.005008000135421753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,128,0.12712639570236206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,64,0.22308480739593506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,64,0.06700800061225891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,16,0.018508799374103546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,32,0.05275840163230896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,32,0.11543680429458618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,128,0.43829760551452634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,4,0.016956800222396852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,8,0.026025599241256712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,8,0.008348800241947174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,1536,2,0.012374400347471236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,16,0.060703998804092406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,1,0.005020799860358238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,4,0.005833600088953972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,2048,2,0.005100800096988678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,128,0.1850335955619812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,128,0.6524928092956543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,8,0.03237760066986084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,32,0.1684671998023987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,64,0.09543359875679017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,2,0.013952000439167023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,4,0.01971839964389801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,32,0.051183998584747314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,2048,1,0.011283200234174728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,16,0.040092799067497256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,64,0.33056321144104006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,8,0.010073599964380264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,4,0.007321599870920181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,8,0.04840959906578064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,2,0.005475199967622757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,16,0.08806399703025818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,64,0.43664960861206054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,3072,1,0.005084799975156784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,128,0.8620896339416504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,128,0.24179201126098632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,64,0.1252671957015991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,32,0.06600000262260437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,32,0.2234208106994629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,16,0.03832640051841736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,8,0.01730560064315796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,4,0.0253248006105423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,1,0.012486399710178375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,3072,2,0.01679680049419403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,4,0.007996799796819687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,16,0.11526399850845337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,8,0.060121601819992064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,2,0.0057599999010562895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,4096,1,0.005110400170087815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,64,0.1806015968322754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,128,0.35685439109802247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,4,0.03173440098762512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,64,0.6504447937011719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,2,0.01945279985666275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,32,0.32941761016845705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,128,1.2842432022094727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,32,0.09535359740257263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,4096,1,0.01395840048789978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,16,0.0564191997051239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,4,0.010086400061845779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,16,0.16826879978179932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,8,0.029164800047874452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,2,0.007302399724721909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,6144,1,0.005596800148487091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,128,0.5086368083953857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,8,0.0877888023853302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,128,1.7083040237426759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,4,0.048860800266265866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,64,0.23921918869018555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,64,0.8601920127868652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,32,0.12333120107650757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,16,0.07335360050201416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,1,0.017340800166130065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,8,0.038649600744247434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,16,0.22267839908599854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,6144,2,0.025382399559020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,2,0.008108799904584884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,4,0.018438400328159334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,32,0.4352735996246338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,8192,1,0.005670399963855743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,128,0.7018239974975586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,8,0.11544640064239502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,32,0.20751678943634033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,128,2.546086311340332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,64,0.37705919742584226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,4,0.060361599922180174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,2,0.031174400448799135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,64,1.2797439575195313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,16,0.09360640048980713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,8192,1,0.01998720020055771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,32,0.6473472118377686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,16,0.3326495885848999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,2,0.010000000149011612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,8,0.049958398938179015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,4,0.027059200406074523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,4,0.0883296012878418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,128,0.9296928405761719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,2,0.04906240105628967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,12288,1,0.007264000177383423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,8,0.16857279539108277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,32,0.24342720508575438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,64,0.4671167850494385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,128,3.3902015686035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,64,1.701033592224121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,16,0.123363196849823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,32,0.9449055671691895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,8,0.06491519808769226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,1,0.008422400057315826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,4,0.03797760009765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,4,0.11606400012969971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,16384,2,0.01584320068359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,2,0.061510401964187625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,12288,1,0.02540160119533539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,16,0.43432960510253904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,8,0.2236543893814087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,128,1.154755210876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,64,0.5796383857727051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,128,4.226153564453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,64,2.3407520294189452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,16384,1,0.03136000037193298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,32,0.30687680244445803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,16,0.15018559694290162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,32,1.0660191535949708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,16,0.5442975997924805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,8,0.0899392008781433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,4,0.07550719976425171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,2,0.02226240038871765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,2,0.07584959864616395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,float16,20480,1,0.008742400258779526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,4,0.14163520336151122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,8,0.27894399166107176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_gen_post,fp8,20480,1,0.04278079867362976
