framework,version,device,op_name,bmm_dtype,num_tokens,num_heads,latency
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,128,0.006054399907588959
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,64,0.004924799874424934
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,32,0.004416000097990036
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,16,0.004588799923658371
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,64,0.01071999967098236
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,128,0.015939199924468996
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,8,0.0040895998477935794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,32,0.008207999914884568
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,16,0.00719040036201477
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,4,0.004032000154256821
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,1,0.00411520004272461
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,8,0.00650240033864975
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,2,0.006111999973654747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,1,0.0055808000266551975
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,64,0.005081599950790406
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1,2,0.008428800106048583
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,128,0.0058432001620531086
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1,4,0.006403200328350067
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,32,0.008470399677753449
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,128,0.016553600132465363
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,16,0.00416640006005764
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,32,0.004252799972891808
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,64,0.010835199803113937
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,8,0.004470400139689445
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,2,0.006364800035953522
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,2,0.004383999854326248
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,16,0.007423999905586243
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,1,0.004092799872159958
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2,4,0.004284799844026565
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,4,0.00684799998998642
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,8,0.006796800345182419
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,128,0.006019200012087822
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2,1,0.005875200033187866
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,32,0.004582399874925614
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,64,0.005107200145721436
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,64,0.011209599673748016
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,16,0.0044319998472929
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,8,0.004419200122356415
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,128,0.016598400473594666
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,32,0.008787199854850769
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,2,0.004502400010824204
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,4,0.005187200009822845
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,16,0.007372800260782242
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4,1,0.00416640006005764
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,128,0.005932800099253654
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,4,0.006864000111818314
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,8,0.0068512000143527985
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,64,0.005027199909090996
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,2,0.006492800265550614
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4,1,0.00594559982419014
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,32,0.004396799951791763
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,16,0.004879999905824661
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,16,0.0076895996928215025
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,128,0.017110399901866913
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,8,0.004416000097990036
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,4,0.004339199885725975
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,64,0.011468800157308579
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,32,0.008691199868917466
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,8,0.006940799951553345
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,2,0.004134399816393852
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,128,0.006111999973654747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8,1,0.004208000004291534
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,64,0.00493439994752407
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,32,0.004521600157022476
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,16,0.004156799986958504
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,4,0.006774400174617767
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,1,0.006268800050020218
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,128,0.01794559955596924
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,8,0.004153599962592125
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8,2,0.0065760001540184024
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,64,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,32,0.008995199948549271
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,8,0.007152000069618225
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,4,0.007097599655389785
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,2,0.004112000018358231
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,16,0.007952000200748443
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,4,0.004956800118088722
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,64,0.005104000121355057
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,16,1,0.004003199934959412
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,128,0.019849599897861482
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,128,0.006345599889755249
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,1,0.006220800057053566
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,16,2,0.006665600091218948
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,32,0.00928959995508194
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,64,0.012582400441169738
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,16,0.004335999861359597
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,32,0.004873599857091904
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,8,0.00658240020275116
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,4,0.007327999919652939
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,2,0.0048351999372243885
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,4,0.0040352001786232
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,128,0.007001599669456482
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,8,0.0074592001736164095
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,16,0.00825600028038025
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,2,0.006969600170850754
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,32,1,0.006563200056552887
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,32,1,0.005033599957823753
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,128,0.021401600539684297
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,64,0.00560000017285347
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,32,0.009641599655151368
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,32,0.004726399853825569
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,16,0.004403200000524521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,8,0.004867200180888176
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,4,0.0040608000010252
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,64,0.01326719969511032
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,2,0.0042304001748561856
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,4,0.007472000271081925
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,2,0.007273600250482559
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,16,0.008457600325345992
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,48,1,0.004950400069355964
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,8,0.007571200281381607
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,128,0.00764160007238388
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,64,0.014668799936771393
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,32,0.004950400069355964
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,64,0.005820799991488457
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,16,0.0046431999653577805
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,48,1,0.006623999774456024
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,8,0.004182400181889534
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,128,0.02260800004005432
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,8,0.007964800298213958
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,4,0.004105599969625473
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,32,0.01021760031580925
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,16,0.008729600161314011
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,2,0.00761599987745285
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,1,0.00721919983625412
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,2,0.0040608000010252
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,64,1,0.004428799822926521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,128,0.008214399963617326
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,64,0.006067200005054474
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,64,4,0.007820799946784973
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,16,0.004495999962091446
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,32,0.005193600058555603
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,8,0.00448639988899231
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,4,0.004291199892759323
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,2,0.0040895998477935794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,64,0.015747199952602386
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,80,1,0.004176000133156776
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,16,0.008928000181913375
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,32,0.010790400207042694
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,128,0.02547839879989624
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,128,0.008636800199747085
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,8,0.008089599758386612
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,4,0.007670400291681289
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,64,0.006371200084686279
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,32,0.005167999863624572
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,2,0.0072063997387886046
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,8,0.0043680001050233844
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,16,0.004848000034689904
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,128,0.025894400477409363
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,80,1,0.007084800302982331
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,4,0.004092799872159958
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,32,0.010969600081443787
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,4,0.00809279978275299
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,2,0.004076800122857094
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,96,1,0.004080000147223472
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,64,0.016332800686359405
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,16,0.009068799763917923
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,8,0.007942400127649307
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,64,0.007344000041484833
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,128,0.02842560112476349
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,128,0.010156799852848054
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,32,0.005497600138187409
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,64,0.017008000612258913
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,2,0.007420799881219864
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,16,0.004675199836492538
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,96,1,0.007152000069618225
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,16,0.009436800330877303
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,8,0.004735999926924706
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,4,0.004924799874424934
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,32,0.011683200299739838
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,2,0.004025600105524063
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,64,0.0081216000020504
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,128,1,0.004131200164556504
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,128,0.012281599640846252
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,8,0.008364800363779068
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,4,0.00788159966468811
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,2,0.007708799839019775
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,32,0.005932800099253654
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,128,0.042080000042915344
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,128,1,0.0076384000480175015
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,16,0.004755200073122978
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,64,0.02288320064544678
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,8,0.004656000062823296
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,4,0.0043680001050233844
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,32,0.014767999947071075
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,2,0.004284799844026565
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,160,1,0.004284799844026565
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,128,0.01636800020933151
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,64,0.024099199473857878
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,16,0.010566399991512298
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,2,0.0077344000339508055
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,8,0.00883520022034645
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,64,0.008377599716186523
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,4,0.008032000064849854
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,160,1,0.007475200295448303
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,32,0.00628800019621849
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,128,0.0450655996799469
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,16,0.004937599971890449
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,8,0.004668800160288811
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,4,0.004569600149989128
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,2,0.007788799703121185
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,32,0.01555200070142746
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,2,0.004086399823427201
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,192,1,0.0042975999414920805
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,16,0.010470400005578995
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,128,0.02127680033445358
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,1,0.007327999919652939
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,8,0.009388799965381622
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,192,4,0.008198399841785432
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,64,0.009641599655151368
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,32,0.007001599669456482
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,8,0.005119999870657921
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,16,0.0053888000547885895
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,4,0.008179199695587159
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,128,0.05039039850234985
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,4,0.004399999976158142
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,2,0.004425600171089172
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,32,0.01578560024499893
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,64,0.026940798759460448
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,16,0.01114879995584488
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,128,0.023849600553512575
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,64,0.012057600170373916
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,8,0.009113600105047226
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,256,1,0.004105599969625473
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,2,0.007910399883985519
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,32,0.0072512000799179075
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,128,0.06815360188484192
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,256,1,0.007536000013351441
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,32,0.020313599705696107
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,16,0.0057183999568223955
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,8,0.004950400069355964
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,64,0.03565439879894257
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,4,0.004735999926924706
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,2,0.004345599934458733
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,320,1,0.004787199944257736
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,1,0.007507199794054032
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,128,0.027497598528862
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,16,0.013036799430847169
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,32,0.008451200276613235
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,64,0.0142752006649971
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,64,0.03750079870223999
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,8,0.009187199920415879
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,2,0.007766400277614593
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,320,4,0.00840959995985031
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,16,0.00615679994225502
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,32,0.020908799767494202
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,128,0.0706816017627716
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,8,0.005104000121355057
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,2,0.004208000004291534
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,4,0.004447999969124794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,8,0.009123200178146362
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,4,0.008479999750852585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,128,0.033795198798179625
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,384,1,0.004307200014591217
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,1,0.007798399776220322
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,16,0.013526399433612824
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,128,0.08993600010871887
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,32,0.009683199971914292
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,64,0.018620799481868743
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,384,2,0.008086399734020233
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,32,0.025673601031303405
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,16,0.006950400024652481
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,8,0.005225599929690361
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,4,0.004652800038456917
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,64,0.047830399870872495
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,16,0.015408000349998474
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,1,0.003996799886226654
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,512,2,0.004441599920392036
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,1,0.007814399898052216
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,128,0.04674560129642487
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,128,0.1302880048751831
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,64,0.026153600215911864
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,8,0.010812799632549285
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,64,0.06893759965896606
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,16,0.008367999643087386
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,4,0.00926079973578453
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,512,2,0.008099199831485748
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,32,0.015455999970436096
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,8,0.006006399914622307
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,4,0.004969599843025208
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,2,0.004681599885225296
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,32,0.036134400963783266
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,768,1,0.004083200171589851
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,16,0.020291200280189513
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,8,0.013219200074672699
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,64,0.03289920091629028
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,128,0.1683583974838257
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,128,0.05931519865989685
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,32,0.0472927987575531
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,4,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,2,0.008470399677753449
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,32,0.018671999871730804
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,8,0.0070720002055168155
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,768,1,0.008038400113582611
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,16,0.009647999703884125
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,16,0.02526400089263916
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,64,0.08802559971809387
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,4,0.005215999856591225
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,2,0.004687999933958053
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1024,1,0.004342399910092354
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,64,0.04586879909038544
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,128,0.24706881046295165
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,128,0.08849599957466125
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,8,0.015408000349998474
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,32,0.025167998671531678
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,64,0.12673280239105225
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,4,0.010860799998044967
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,32,0.06737279891967773
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,16,0.03555200099945068
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,1,0.00841279998421669
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,8,0.008479999750852585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1024,2,0.009151999652385712
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,16,0.013692800700664521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,4,0.006095999851822853
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,1,0.004428799822926521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,1536,2,0.005110400170087815
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,128,0.11383359432220459
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,64,0.05905600190162659
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,8,0.02016959935426712
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,128,0.32724480628967284
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,4,0.013257600367069244
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,64,0.16725759506225585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,32,0.08715199828147888
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,32,0.03169600069522858
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,16,0.01746560037136078
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,8,0.009782399982213974
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,1,0.008479999750852585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,8,0.025414401292800905
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,4,0.006803199648857117
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,1536,2,0.009193599969148637
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,128,0.17418559789657592
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,2,0.005283199995756149
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,16,0.04684160053730011
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,2048,1,0.0049632001668214795
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,64,0.08396160006523132
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,4,0.015161600708961488
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,32,0.12668479681015016
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,64,0.24215359687805177
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,32,0.044633600115776065
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,128,0.47945599555969237
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,16,0.024476799368858337
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,16,0.06622719764709473
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,2,0.01072319969534874
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,8,0.01401280015707016
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,2048,1,0.00910080000758171
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,8,0.03495039939880371
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,4,0.008428800106048583
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,2,0.006089600175619126
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,3072,1,0.005036799982190132
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,64,0.3211040019989014
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,128,0.23107519149780273
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,128,0.6269951820373535
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,64,0.11031359434127808
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,4,0.019724799692630766
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,32,0.16461440324783325
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,32,0.057545602321624756
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,2,0.013081599771976472
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,16,0.03150399923324585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,4,0.0095551997423172
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,16,0.08602560162544251
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,8,0.017654399573802947
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,3072,1,0.009001599997282029
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,1,0.0054687999188899996
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,4096,2,0.00692799985408783
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,128,0.36010239124298093
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,8,0.04594239890575409
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,64,0.16622079610824586
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,4,0.024726399779319765
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,128,0.9302304267883301
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,64,0.4713888168334961
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,32,0.08292800188064575
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,16,0.04460160136222839
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,2,0.015039999783039094
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,8,0.026015999913215637
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,4096,1,0.010838399827480315
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,4,0.013900800049304963
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,2,0.008297599852085114
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,32,0.24196159839630127
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,16,0.12593599557876586
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,6144,1,0.006252799928188324
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,8,0.06584640145301819
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,4,0.03460479974746704
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,2,0.02022400051355362
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,128,0.4802815914154053
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,128,1.2093471527099608
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,64,0.21839361190795897
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,16,0.056892800331115725
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,64,0.6229440212249756
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,32,0.3193631887435913
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,32,0.10818239450454711
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,6144,1,0.012956799566745758
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,8,0.03399679958820343
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,16,0.1639232039451599
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,4,0.01733759939670563
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,2,0.009750399738550186
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,float16,8192,1,0.0069311998784542085
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,8,0.08575680255889892
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,4,0.04589439928531647
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,2,0.02441920042037964
TRTLLM,0.20.0,NVIDIA H200,mla_gen_pre,fp8,8192,1,0.015471999347209931
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,128,0.005859199911355972
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,128,0.011785600334405899
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,8,0.006844799965620041
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,4,0.004630399867892265
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,32,0.004694399982690811
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,64,0.005046400055289268
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,2,0.004284799844026565
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,64,0.00947519987821579
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,16,0.004454400017857551
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,32,0.008134400099515915
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,1,0.0042720001190900804
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1,8,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,4,0.006745599955320358
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,16,0.007670400291681289
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,32,0.004793599992990494
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,64,0.005062400177121162
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,128,0.005465599894523621
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,2,0.00634239986538887
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,128,0.011670400202274323
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,32,0.008153600245714187
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,16,0.004479999840259552
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1,1,0.006140799820423126
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,64,0.009510400146245957
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,8,0.005632000043988228
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,2,0.0043680001050233844
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,4,0.004278400167822838
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2,1,0.004403200000524521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,16,0.008294399827718735
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,4,0.006569600105285645
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,8,0.006883200258016586
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,128,0.012377600371837615
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,128,0.005577600002288819
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,2,0.006537599861621857
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,64,0.005087999999523163
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2,1,0.006249599903821945
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,32,0.004678399860858917
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,16,0.004387199878692627
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,64,0.00981760025024414
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,4,0.00432640016078949
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,4,0.006969600170850754
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,32,0.008249600231647492
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,8,0.00639680027961731
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,16,0.007731200009584427
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,2,0.004227200150489807
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,8,0.007248000055551529
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4,1,0.004383999854326248
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,128,0.005750399827957153
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,64,0.0047520000487566
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,1,0.006351999938488007
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4,2,0.0068063996732234955
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,32,0.004704000055789947
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,16,0.004387199878692627
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,128,0.013670399785041809
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,8,0.004303999990224838
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,16,0.007865600287914276
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,64,0.009971199929714203
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,4,0.004550400003790855
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,32,0.008495999872684479
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,2,0.004303999990224838
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8,1,0.004419200122356415
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,128,0.006012799963355064
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,8,0.007023999840021134
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,2,0.006636799871921539
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,64,0.004873599857091904
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,4,0.006892800331115723
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,16,0.004380799829959869
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8,1,0.006470400094985962
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,32,0.004678399860858917
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,128,0.014256000518798828
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,8,0.0047520000487566
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,4,0.004464000090956688
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,2,0.004320000112056732
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,64,0.011030399799346923
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,8,0.007283200323581695
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,16,0.007807999849319458
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,2,0.0068960003554821014
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,32,0.00904960036277771
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16,1,0.004268800094723702
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,1,0.0067391999065876005
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16,4,0.007532799988985062
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,128,0.0064351998269557955
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,64,0.005129599943757057
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,32,0.004771199822425842
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,32,0.0100832000374794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,16,0.0045471999794244765
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,128,0.01733759939670563
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,8,0.004396799951791763
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,64,0.01239679977297783
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,4,0.004300799965858459
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,2,0.007500799745321274
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,2,0.004425600171089172
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,32,1,0.004380799829959869
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,128,0.02041279971599579
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,8,0.0075392000377178196
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,128,0.006566400080919266
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,64,0.005302400141954422
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,4,0.007391999661922455
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,16,0.008675199747085572
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,32,0.004687999933958053
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,64,0.014022399485111237
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,32,1,0.0068960003554821014
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,4,0.004339199885725975
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,16,0.004652800038456917
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,8,0.004572800174355507
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,2,0.0075552001595497135
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,32,0.01114879995584488
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,4,0.00783040001988411
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,2,0.004438399896025657
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,8,0.008348800241947174
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,16,0.008928000181913375
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,48,1,0.007353600114583969
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,48,1,0.004332799836993218
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,128,0.006982400268316269
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,64,0.005452800169587135
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,8,0.004704000055789947
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,128,0.02306240051984787
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,64,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,32,0.011500799655914306
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,16,0.0049632001668214795
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,16,0.010044799745082855
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,4,0.004444799944758415
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,32,0.005488000065088272
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,8,0.008656000345945358
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,2,0.004387199878692627
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,64,1,0.004345599934458733
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,128,0.007433599978685379
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,64,0.00562559999525547
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,32,0.004950400069355964
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,2,0.007753600180149078
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,4,0.007974400371313094
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,64,1,0.007388799637556076
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,16,0.004828799888491631
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,8,0.004518400132656098
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,4,0.004390399903059006
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,2,0.004335999861359597
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,32,0.012352000176906585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,64,0.01765120029449463
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,128,0.027910399436950683
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,16,0.010380800068378448
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,80,1,0.004355200007557869
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,8,0.00868159979581833
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,128,0.008079999685287475
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,128,0.031145599484443665
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,64,0.00584000013768673
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,4,0.007833600044250488
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,2,0.007628799974918365
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,32,0.004838399961590767
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,16,0.0055263999849557875
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,80,1,0.007465600222349167
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,64,0.01873600035905838
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,8,0.004662400111556053
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,16,0.010665600001811982
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,4,0.004479999840259552
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,2,0.004422400146722794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,32,0.012988799810409546
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,96,1,0.004454400017857551
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,8,0.008694399893283845
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,128,0.008607999980449676
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,64,0.006291200220584869
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,2,0.0077855996787548065
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,1,0.007795199751853943
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,96,4,0.008473599702119828
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,128,0.03654719889163971
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,16,0.005008000135421753
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,32,0.005023999884724617
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,64,0.021590399742126464
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,8,0.004870399832725525
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,4,0.004704000055789947
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,2,0.004275200143456459
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,32,0.014659200608730317
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,16,0.01135680004954338
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,1,0.007926400005817413
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,128,1,0.004428799822926521
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,4,0.008764799684286118
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,8,0.010127999633550645
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,128,2,0.00840959995985031
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,128,0.010313600301742554
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,64,0.006585600227117539
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,32,0.005382400006055832
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,16,0.005382400006055832
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,128,0.046902400255203244
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,8,0.0047520000487566
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,4,0.00910400003194809
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,4,0.00459199994802475
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,1,0.006166400015354156
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,8,0.01055999994277954
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,160,2,0.004572800174355507
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,2,0.008422400057315826
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,1,0.007865600287914276
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,64,0.025222399830818178
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,32,0.016655999422073364
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,128,0.014131200313568116
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,160,16,0.012374400347471236
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,32,0.005548800155520439
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,64,0.006985600292682648
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,16,0.004940799996256828
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,8,0.004639999940991402
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,128,0.05160319805145264
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,32,0.017849600315093993
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,4,0.004582399874925614
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,64,0.02786880135536194
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,2,0.004460800066590309
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,16,0.0130048006772995
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,4,0.009008000046014786
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,2,0.008579199761152267
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,192,1,0.004348799958825112
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,8,0.010809600353240967
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,64,0.007804799824953079
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,128,0.018886399269104005
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,32,0.005958399921655655
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,32,0.02069759964942932
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,16,0.005071999877691269
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,16,0.014390400052070618
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,192,1,0.008207999914884568
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,128,0.0634112000465393
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,4,0.0047231998294591905
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,8,0.01085439994931221
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,8,0.004675199836492538
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,64,0.03409599959850311
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,2,0.005423999950289726
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,1,0.007910399883985519
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,256,1,0.004454400017857551
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,128,0.021532799303531646
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,32,0.006595200300216675
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,64,0.008620800077915191
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,4,0.009769599884748459
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,16,0.0052928000688552855
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,256,2,0.008505599945783615
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,128,0.07889599800109863
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,8,0.005046400055289268
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,64,0.044336000084877016
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,4,0.004633599892258644
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,2,0.0046847999095916745
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,320,1,0.005091200023889542
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,128,0.09093760251998902
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,128,0.024265600740909575
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,32,0.023932799696922302
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,4,0.010505600273609162
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,16,0.01603199988603592
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,8,0.011990399658679962
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,1,0.008217599987983704
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,64,0.009644799679517747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,320,2,0.008931200206279754
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,64,0.04940159916877747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,32,0.006604799628257751
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,16,0.005344000086188317
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,8,0.004937599971890449
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,4,0.005270399898290634
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,2,0.004595199972391129
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,16,0.01759999990463257
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,32,0.026473599672317504
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,384,1,0.0045471999794244765
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,8,0.012511999905109405
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,4,0.010771200060844421
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,128,0.030377599596977233
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,32,0.007859200239181519
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,2,0.009142400324344635
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,64,0.01595519930124283
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,384,1,0.008643200248479843
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,8,0.005104000121355057
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,16,0.0056928001344203946
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,128,0.11824959516525269
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,64,0.06178560256958008
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,4,0.004921599850058555
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,2,0.00459199994802475
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,32,0.03243519961833954
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,16,0.020729599893093108
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,8,0.013961599767208099
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,512,1,0.004659200087189674
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,1,0.008371199667453765
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,64,0.023052799701690673
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,4,0.010995200276374817
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,128,0.04195519983768463
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,128,0.17047359943389892
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,512,2,0.009673599898815156
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,32,0.009276799857616425
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,16,0.006675200164318084
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,8,0.0054143998771905896
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,2,0.005203200131654739
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,4,0.004982399940490723
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,64,0.08914560079574585
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,768,1,0.005289600044488907
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,32,0.04909439980983734
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,2,0.010380800068378448
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,128,0.05316479802131653
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,1,0.008956799656152726
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,128,0.22432000637054444
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,32,0.012297599762678146
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,8,0.017148800194263458
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,4,0.012220799922943115
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,16,0.0078015998005867004
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,768,16,0.02648639976978302
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,64,0.029104000329971312
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,4,0.0052352000027894975
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,64,0.11619199514389038
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,8,0.005769599974155426
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,32,0.061212801933288576
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,2,0.0050432000309228895
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1024,1,0.0046016000211238865
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,4,0.013804799318313599
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,16,0.03187839984893799
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,128,0.07500159740447998
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,8,0.01993599981069565
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,2,0.011081600189208984
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,64,0.16861120462417603
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,64,0.04063999950885773
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,32,0.08837119936943054
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,32,0.021929599344730377
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,16,0.00904960036277771
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,128,0.32959039211273194
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,8,0.006550399959087372
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1024,1,0.009948799759149552
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,4,0.005475199967622757
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,1,0.004825599864125252
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,1536,2,0.004940799996256828
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,16,0.04815999865531921
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,128,0.43831357955932615
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,128,0.0970624029636383
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,1,0.01048320010304451
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,64,0.05215680003166199
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,32,0.02757439911365509
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,64,0.222489595413208
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,8,0.025993600487709045
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,2,0.012163200229406358
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,32,0.11451840400695801
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,16,0.011452800035476685
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,1536,4,0.016835199296474458
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,8,0.0077632002532482145
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,4,0.00565119981765747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,2,0.005164799839258194
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,2048,1,0.004825599864125252
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,16,0.060048002004623416
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,8,0.031481599807739256
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,128,0.14171839952468873
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,4,0.019801600277423857
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,2,0.01383039951324463
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,128,0.6494463920593262
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,2048,1,0.010828799754381179
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,64,0.3283488035202026
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,32,0.040249601006507874
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,64,0.07347519993782044
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,32,0.16862720251083374
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,16,0.021030400693416596
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,8,0.009302400052547455
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,4,0.0063680000603199005
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,2,0.005299200117588043
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,3072,1,0.005353600159287453
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,8,0.047391998767852786
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,16,0.08745279908180237
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,128,0.19150079488754274
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,128,0.8593184471130371
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,64,0.43534398078918457
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,64,0.09452800154685974
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,32,0.050886398553848265
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,16,0.11420799493789673
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,4,0.02600640058517456
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,16,0.028915199637413024
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,1,0.012220799922943115
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,3072,2,0.01706559956073761
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,8,0.011334399878978729
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,32,0.22140800952911377
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,4,0.0076895996928215025
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,4,0.031600001454353335
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,2,0.0056639999151229855
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,4096,1,0.004985599964857102
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,128,1.278444766998291
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,128,0.27992639541625974
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,8,0.05995839834213257
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,64,0.138864004611969
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,2,0.01960960030555725
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,32,0.07215359807014465
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,64,0.6476863861083985
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,16,0.04222719967365265
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,8,0.022777600586414336
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,4096,1,0.013625599443912506
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,16,0.16747839450836183
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,32,0.32842559814453126
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,4,0.009177599847316743
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,2,0.006390400230884552
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,6144,1,0.0055263999849557875
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,128,0.3956736087799072
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,8,0.087254399061203
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,64,0.17988159656524658
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,4,0.04751999974250794
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,128,1.6993824005126954
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,32,0.09360640048980713
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,64,0.8567616462707519
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,8,0.1149183988571167
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,16,0.053478401899337766
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,32,0.4328159809112549
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,16,0.22075200080871582
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,8,0.028815999627113342
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,2,0.025593599677085875
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,6144,1,0.01687999963760376
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,4,0.01138240024447441
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,2,0.007644800096750259
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,128,0.526700782775879
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,8192,1,0.00565119981765747
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,128,2.518886375427246
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,64,0.2784991979598999
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,4,0.05950080156326294
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,32,0.1471328020095825
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,2,0.03131519854068756
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,64,1.271615982055664
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,8192,1,0.019382399320602418
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,32,0.6455359935760498
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,8,0.16693439483642578
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,8,0.0385343998670578
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,16,0.07657920122146607
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,4,0.020476800203323365
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,2,0.009030400216579438
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,12288,1,0.0067103996872901915
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,16,0.32724800109863283
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,64,1.6930463790893555
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,128,0.7749695777893066
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,128,3.3513153076171873
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,4,0.08749439716339111
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,64,0.35158720016479494
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,2,0.04696640074253082
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,16,0.09951679706573487
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,32,0.17923200130462646
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,8,0.2219615936279297
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,8,0.050425601005554196
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,4,0.02659519910812378
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,32,0.8580863952636719
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,2,0.011129599809646607
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,12288,1,0.025491198897361754
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,16384,1,0.007676800340414047
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,16,0.4345983982086182
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,128,0.9083264350891114
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,4,0.11368000507354736
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,64,0.4646495819091797
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,32,0.23265600204467773
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,64,2.099897575378418
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,2,0.059894400835037234
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,32,1.065443229675293
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,128,4.179961776733398
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,16384,1,0.03142400085926056
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,16,0.11497600078582763
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,16,0.5395999908447265
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,4,0.14048320055007935
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,8,0.06101760268211365
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,4,0.03246079981327057
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,2,0.01879040002822876
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,2,0.07597439885139465
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,1,0.04140160083770752
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,float16,20480,1,0.008601599931716919
TRTLLM,0.20.0,NVIDIA H200,mla_gen_post,fp8,20480,8,0.27410240173339845
