framework,version,device,op_name,mla_dtype,kv_cache_dtype,batch_size,isl,tp_size,step,latency
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,1,0.021712000171343487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,1,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,1,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,1,0.021669333179791767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,1,0.027269333600997925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,1,0.02717333287000656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,1,0.025642665723959606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,1,0.026799999177455902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,3,0.022597332795461018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,3,0.028069332242012024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,1,0.028730665644009907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,1,0.025936000049114227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,1,0.025850666066010792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,3,0.027109332382678986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,1,0.025786665578683216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,1,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,3,0.02703999976317088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,3,0.02531733363866806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,1,0.025744001070658367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,3,0.022469334304332733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,3,0.025722667574882507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,3,0.025626666843891144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,3,0.026895999908447266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,3,0.026426665484905243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,3,0.02701866626739502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,3,0.02532800038655599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,3,0.026208000878492992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,7,0.021920000513394673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,3,0.02550400048494339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,7,0.027034667630990345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,7,0.025946666797002155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,7,0.026005332668622334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,7,0.026549334327379864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,7,0.02573866645495097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,7,0.02606400102376938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,7,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,7,0.02696000039577484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,7,0.022517333428064983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,7,0.0262773334980011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,7,0.026234666506449383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,7,0.025834667185942333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,7,0.026554666459560394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,15,0.022101332743962605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,15,0.027327999472618103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,15,0.026698666314284008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,15,0.025770666698614757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,15,0.026261332134405773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,15,0.0259253333012263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,15,0.025450666745503742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,15,0.021930667261282604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,15,0.027327999472618103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,15,0.026741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,15,0.026170666019121807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,15,0.025626666843891144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,15,0.025706666211287182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,15,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,31,0.021429332594076794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,31,0.027141332626342773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,31,0.02619733413060506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,31,0.025386666258176167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,31,0.025839999318122864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,31,0.025055999557177227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,31,0.025248001019159954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,31,0.02176533391078313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,31,0.02649066597223282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,31,0.026560001075267792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,31,0.025674665967623394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,31,0.025519999365011852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,31,0.02536533276240031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,31,0.025013332565625507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,63,0.02605866640806198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,63,0.02080533280968666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,63,0.024826665719350178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,63,0.024341332415739696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,63,0.025114665428797405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,63,0.0244159996509552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,63,0.02443733314673106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,63,0.02049066623051961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,63,0.025226667523384094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,63,0.026021334032217663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,63,0.024506665766239166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,63,0.024319998919963837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,63,0.02475200096766154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,63,0.024656000236670177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,127,0.023472001155217487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,127,0.028351999819278717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,127,0.027269333600997925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,127,0.027669332921504974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,127,0.026144000391165417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,127,0.02629866699377696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,127,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,127,0.02367999901374181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,127,0.028954667349656422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,127,0.027024000883102417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,127,0.025909334421157837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,127,0.026586666703224182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,127,0.027274665733178455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,127,0.026447998980681103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,255,0.02327466756105423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,255,0.02863466739654541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,255,0.027653334041436512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,255,0.026416001220544178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,255,0.026309333741664886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,255,0.02643200010061264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,255,0.023525332411130268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,255,0.028853334486484528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,255,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,255,0.026880001028378803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,255,0.026949333647886913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,255,0.026320000489552815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,255,0.026250667870044708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,255,0.026496000587940216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,511,0.024085332949956257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,511,0.029365333418051403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,511,0.027776000400384266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,511,0.027306665976842243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,511,0.02696000039577484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,511,0.026682667434215546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,511,0.026565333207448322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,511,0.02932800104220708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,511,0.024090667565663654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,511,0.02718399961789449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,511,0.027119999130566914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,511,0.028117333849271137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,511,0.026752000053723652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,1023,0.030495998760064442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,511,0.02672533442576726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,1023,0.02595199892918269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,1023,0.02887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,1023,0.028165332973003387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,1023,0.02829866607983907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,1023,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,1023,0.026122666895389557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,1023,0.027679999669392902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,1023,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,1023,0.02903999884923299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,1023,0.028373333315054577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,1023,0.02824000020821889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,1023,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,1023,0.02792000025510788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,2047,0.028410665690898895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,2047,0.033610666791598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,2047,0.030453334252039593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,2047,0.031018666923046112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,2047,0.03164800008138021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,2047,0.030389333764712017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,2047,0.030586667358875275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,2047,0.028501334289709728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,2047,0.033733333150545754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,2047,0.030346666773160298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,2047,0.03170666595300039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,2047,0.029850666721661884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,2047,0.030917334059874218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,2047,0.03017599880695343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,4095,0.030415999392668407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,4095,0.03823466598987579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,4095,0.03615466753641764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,4095,0.0349386657277743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,4095,0.03425599883000056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,4095,0.034645333886146545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,4095,0.02978666623433431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,4095,0.03825599948565165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,4095,0.03451200077931086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,4095,0.036464000741640724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,4095,0.03510399907827377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,4095,0.03439466655254364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,4095,0.03455466777086258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,4095,0.034304000437259674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,8191,0.04081599911053976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,8191,0.03382933388153712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,8191,0.037808001041412354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,8191,0.037087999284267426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,8191,0.03738666574160258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,8191,0.03849600007136663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,8191,0.037477334340413414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,8191,0.033941333492596946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,8191,0.04082666585842768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,8191,0.03808533400297165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,8191,0.037776000797748566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,8191,0.03714133302370707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,8191,0.0373333344856898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,16383,0.050586665670077004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,16383,0.04074133435885111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,8191,0.03735466549793879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,16383,0.04731200138727824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,16383,0.04599999884764353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,16383,0.045269335309664406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,16383,0.045653333266576133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,16383,0.04497066636880239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,16383,0.03951466580231985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,16383,0.051034669081370033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,16383,0.04744000236193339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,16383,0.04638933142026266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,16383,0.04586666822433472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,16383,0.04533333579699198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,16383,0.04541866481304169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,32767,0.06271466612815857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,32767,0.058490668733914696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,32767,0.061797335743904114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,32767,0.05346133311589559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,32767,0.05070933202902476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,32767,0.04996799925963084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,32767,0.05055999755859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,32767,0.05156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,32767,0.07144533097743988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,32767,0.05292266607284546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,32767,0.058415999015172325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,32767,0.050944000482559204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,32767,0.0498986691236496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,32767,0.05011733373006185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,65535,0.06754133105278015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,65535,0.07716266810894012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,65535,0.08919466535250346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,65535,0.06965333223342896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,65535,0.0691786656777064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,65535,0.06850133339564006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,65535,0.06791466474533081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,65535,0.06820266445477803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,65535,0.07663999994595845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,65535,0.07197333375612895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,65535,0.08635733524958293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,1,131071,0.09522666533788045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,65535,0.06900266806284587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,65535,0.06866133213043213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,65535,0.06829333305358887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,2,131071,0.10116799672444661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,4,131071,0.09704533219337463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,8,131071,0.09527466694513957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,16,131071,0.12345066666603088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,32,131071,0.09258666634559631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,1,131071,0.09619200229644775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1,1,64,131071,0.09263466795285542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,2,131071,0.10156266887982686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,4,131071,0.09575466314951579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,8,131071,0.09419199824333191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,16,131071,0.09313600262006123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,32,131071,0.12274666627248128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1,1,64,131071,0.09145599603652954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,1,0.027509334186712902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,1,0.02842666705449422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,1,0.023082666099071503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,1,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,1,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,1,0.02681066592534383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,1,0.023034666975339253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,1,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,1,0.028234665592511494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,1,0.027621333797772724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,1,0.026613332331180573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,1,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,1,0.026373334228992462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,3,0.023056000471115112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,1,0.026874666412671406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,3,0.02802666773398717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,3,0.027050666511058807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,3,0.027050666511058807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,3,0.026122666895389557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,3,0.026496000587940216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,3,0.02310933421055476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,3,0.02624000112215678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,3,0.028394666810830433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,3,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,3,0.026693334182103474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,3,0.026549334327379864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,3,0.026122666895389557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,3,0.02644266684850057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,7,0.022986667851607006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,7,0.027535999814669292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,7,0.02749866743882497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,7,0.026842666169007618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,7,0.026309333741664886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,7,0.02589866767326991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,7,0.026309333741664886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,7,0.022805333137512207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,7,0.02773333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,7,0.02683199942111969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,7,0.0262719988822937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,7,0.027066667874654133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,7,0.02624000112215678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,7,0.02587733417749405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,15,0.022453332940737408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,15,0.027471999327341717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,15,0.02665599932273229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,15,0.025946666797002155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,15,0.02603200078010559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,15,0.0262773334980011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,15,0.02236266682545344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,15,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,15,0.026000000536441803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,15,0.026789332429567974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,15,0.0262719988822937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,15,0.025653332471847534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,15,0.025888000925381977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,15,0.02632533262173335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,31,0.02629866699377696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,31,0.027066667874654133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,31,0.021984001000722248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,31,0.025685332715511322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,31,0.025125332176685333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,31,0.025461333493391674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,31,0.025594666600227356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,31,0.027024000883102417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,31,0.02145066608985265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,31,0.025653332471847534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,31,0.02649066597223282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,31,0.025461333493391674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,31,0.02526933451493581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,31,0.02548266698916753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,63,0.021061333517233532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,63,0.026159999271233875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,63,0.025610665480295818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,63,0.02470933397610982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,63,0.02489600082238515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,63,0.024336000283559162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,63,0.024314666787783306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,63,0.021136000752449036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,63,0.02605333427588145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,63,0.025216000775496166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,63,0.025216000775496166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,63,0.024319998919963837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,63,0.024469333390394848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,63,0.02477866659561793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,127,0.02369600037733714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,127,0.029274667302767437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,127,0.027866666515668232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,127,0.027066667874654133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,127,0.027104000250498455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,127,0.026496000587940216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,127,0.026533332963784535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,127,0.029157333076000214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,127,0.023685333629449207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,127,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,127,0.02741333345572154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,127,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,255,0.029232000311215717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,255,0.024175999065240223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,127,0.026752000053723652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,127,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,255,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,255,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,255,0.0268053337931633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,255,0.026704000929991405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,255,0.02679466704527537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,255,0.024160000185171764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,255,0.029450667401154835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,255,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,255,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,255,0.02700799951950709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,255,0.026549334327379864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,511,0.02535466601451238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,255,0.026693334182103474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,511,0.030239999294281006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,511,0.028751999139785767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,511,0.028037334481875103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,511,0.0276053324341774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,511,0.027488000690937042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,511,0.025765334566434223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,511,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,511,0.030165334542592365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,511,0.028586665789286297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,511,0.027813332776228588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,511,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,511,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,511,0.027424000203609467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,1023,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,1023,0.03286933402220408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,1023,0.031290667752424874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,1023,0.030037333567937214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,1023,0.02951466788848241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,1023,0.029482667644818623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,1023,0.029696000119050343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,1023,0.028090665737787884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,1023,0.032618666688601174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,1023,0.03108799954255422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,1023,0.029445332785447437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,1023,0.03032533327738444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,1023,0.029322666426499683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,1023,0.02934933453798294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,2047,0.029994666576385498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,2047,0.03661333272854487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,2047,0.03425066669782003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,2047,0.03312533348798752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,2047,0.03271999955177307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,2047,0.03266666581233343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,2047,0.03244800120592117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,2047,0.02962133288383484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,2047,0.03461866577466329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,2047,0.036373332142829895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,2047,0.033344000577926636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,2047,0.03264000018437704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,2047,0.032746667663256325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,2047,0.032602667808532715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,4095,0.03296533226966858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,4095,0.036890665690104164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,4095,0.03879466652870178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,4095,0.03580799947182337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,4095,0.03565866748491923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,4095,0.03545066714286804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,4095,0.03532800078392029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,4095,0.03292266776164373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,4095,0.03825599948565165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,4095,0.036703998843828835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,4095,0.03603200117746989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,4095,0.03543466577927271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,4095,0.035386666655540466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,4095,0.035461333890755974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,8191,0.03804266701141993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,8191,0.04337066908677419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,8191,0.040234667559464775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,8191,0.04101866732041041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,8191,0.040549332896868386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,8191,0.040063999593257904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,8191,0.0399893323580424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,8191,0.03839999934037527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,8191,0.04302933315436045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,8191,0.04159466673930486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,8191,0.04033066580692927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,8191,0.03949866692225138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,8191,0.04022933294375738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,8191,0.03982933362325033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,16383,0.05080533524354299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,16383,0.057722667853037514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,16383,0.06366933385531108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,16383,0.05049066742261251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,16383,0.04598933458328247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,16383,0.045237332582473755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,16383,0.046207999189694725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,16383,0.05148266752560934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,16383,0.057861333092053734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,16383,0.05346133311589559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,16383,0.047295997540156044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,16383,0.05390933156013489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,16383,0.04637333254019419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,16383,0.045370668172836304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,32767,0.06593599915504456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,32767,0.06815466781457265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,32767,0.07356800138950348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,32767,0.06638933221499126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,32767,0.06467199822266896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,32767,0.08076799909273784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,32767,0.06530133386452992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,32767,0.06624533236026764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,32767,0.06825066606203715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,32767,0.07403199871381123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,32767,0.0669653316338857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,32767,0.06539200246334076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,32767,0.06516266862551372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,32767,0.0802400012811025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,65535,0.09487467010815938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,65535,0.09681600332260132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,65535,0.09346666932106018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,65535,0.09147199988365173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,65535,0.08930666248003642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,65535,0.08822932839393616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,65535,0.08892266949017842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,65535,0.12387733658154805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,65535,0.09775466720263164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,65535,0.09204266468683879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,65535,0.0909440020720164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,65535,0.09019200007120769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,65535,0.0885653297106425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,65535,0.08815999825795491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,1,131071,0.15079466501871744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,4,131071,0.1421013375123342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,2,131071,0.21047999461491904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,16,131071,0.13805333773295084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,8,131071,0.1378613313039144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,32,131071,0.13715733091036478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,2,1,64,131071,0.13646399974822998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,1,131071,0.15132266283035278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,2,131071,0.14614933729171753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,16,131071,0.1384106675783793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,8,131071,0.1995519995689392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,4,131071,0.1413706640402476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,32,131071,0.1362826625506083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,2,1,64,131071,0.1372213363647461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,1,0.023567999402681988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,1,0.028255999088287354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,1,0.027637332677841187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,1,0.02682666728893916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,1,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,1,0.026895999908447266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,1,0.026709333062171936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,1,0.022890667120615642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,1,0.028304000695546467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,1,0.02752533306678136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,1,0.026975999275843304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,1,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,1,0.026799999177455902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,1,0.02700799951950709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,3,0.02325333406527837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,3,0.0284853329261144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,3,0.027610667049884796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,3,0.027087998886903126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,3,0.026789332429567974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,3,0.02664533257484436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,3,0.026736001173655193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,3,0.023530667026837666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,3,0.028202667832374573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,3,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,3,0.02771199991305669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,3,0.02731200059254964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,3,0.026741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,3,0.0266239990790685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,7,0.028138667345046997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,7,0.023311999936898548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,7,0.027317332724730175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,7,0.027061333258946735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,7,0.026901334524154663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,7,0.026848000784715016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,7,0.026672000686327618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,7,0.02334933231274287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,7,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,7,0.027989332874615986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,7,0.02677333354949951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,7,0.026682667434215546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,15,0.022815999885400135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,7,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,7,0.026858667532602947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,15,0.027669332921504974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,15,0.027072000006834667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,15,0.026485333840052288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,15,0.026122666895389557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,15,0.02621866762638092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,15,0.026181332767009735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,15,0.028064000109831493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,15,0.0229066660006841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,15,0.02701333413521449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,15,0.026661333938439686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,15,0.026213333010673523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,15,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,15,0.02609066665172577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,31,0.022143999735514324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,31,0.027749332288901012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,31,0.026528000831604004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,31,0.026149332523345947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,31,0.02568000058333079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,31,0.025722667574882507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,31,0.022410665949185688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,31,0.02554133286078771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,31,0.027072000006834667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,31,0.026234666506449383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,31,0.02701866626739502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,31,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,31,0.02589333305756251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,63,0.02130666623512904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,31,0.02569066733121872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,63,0.025610665480295818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,63,0.0260959987839063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,63,0.02510400116443634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,63,0.02510933329661687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,63,0.024549332757790882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,63,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,63,0.021349333226680756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,63,0.0249439999461174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,63,0.026191999514897663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,63,0.025018667181332905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,63,0.02499733368555705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,63,0.0249493345618248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,63,0.024522667129834492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,127,0.02402133246262868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,127,0.02958400050799052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,127,0.02810666710138321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,127,0.027589333554108936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,127,0.02735466758410136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,127,0.027130665878454845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,127,0.027136000494162243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,127,0.02420799930890401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,127,0.02941333254178365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,127,0.02829866607983907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,127,0.027658666173617046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,127,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,127,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,255,0.028618666032950085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,255,0.030218665798505146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,255,0.02609066665172577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,127,0.026885333160559338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,255,0.027989332874615986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,255,0.02756800005833308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,255,0.027258666853109997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,255,0.027072000006834667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,255,0.025477332373460133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,255,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,255,0.030245333909988403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,255,0.02874133239189784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,255,0.027999999622503918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,255,0.027295999228954315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,255,0.02733866622050603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,511,0.027434666951497395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,511,0.032032000521818794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,511,0.029834667841593426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,511,0.029215998947620392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,511,0.029279999434947968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,511,0.030799999833106995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,511,0.02808533360560735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,511,0.02917333443959554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,511,0.032287999987602234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,511,0.03047466774781545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,511,0.028922667105992634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,511,0.02926933268706004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,511,0.029088000456492107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,511,0.029882666965325672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,1023,0.02980799973011017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,1023,0.03550933301448822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,1023,0.03315199911594391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,1023,0.03257599969704946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,1023,0.031856000423431396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,1023,0.03181333343187968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,1023,0.029989334444204967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,1023,0.031557333966096245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,1023,0.03542399903138479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,1023,0.033626665671666466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,1023,0.03197333216667175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,1023,0.03211733450492223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,1023,0.0316746657093366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,2047,0.032645332316557564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,1023,0.031632001201311745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,2047,0.03809600075085958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,2047,0.03585066646337509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,2047,0.03513066718975703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,2047,0.03477866699298223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,2047,0.03417066733042399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,2047,0.03449599941571554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,2047,0.032773333291212715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,2047,0.03602666656176249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,2047,0.03809066613515218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,2047,0.034874667723973594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,2047,0.03473066786924998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,2047,0.034645333886146545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,2047,0.034245334565639496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,4095,0.03766400118668874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,4095,0.042725334564844765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,4095,0.03896533449490865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,4095,0.04098133246103922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,4095,0.039520000418027244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,4095,0.0391893337170283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,4095,0.03975466638803482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,4095,0.03763733307520548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,4095,0.043040002385775246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,4095,0.04099733382463455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,4095,0.038975998759269714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,4095,0.03974399964014689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,4095,0.03948266555865606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,4095,0.038933334251244865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,8191,0.05184000233809153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,8191,0.0547626664241155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,8191,0.05117866893609365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,8191,0.04580266773700714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,8191,0.044778664906819664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,8191,0.04524266719818115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,8191,0.05242133140563965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,8191,0.04409599800904592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,8191,0.054192001620928444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,8191,0.05080533524354299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,8191,0.04496000210444132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,8191,0.0439573327700297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,16383,0.06550933420658112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,8191,0.04486933350563049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,8191,0.04437866806983948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,16383,0.09085866808891296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,16383,0.06724800169467926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,16383,0.06413333117961884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,16383,0.0632533331712087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,16383,0.06354133288065593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,16383,0.06272533535957336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,16383,0.06645333270231883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,16383,0.07221333185831706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,16383,0.08423999945322673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,16383,0.0646666685740153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,16383,0.06340266764163971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,16383,0.0631573349237442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,32767,0.0934986670811971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,16383,0.06305600206057231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,32767,0.09115200241406758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,32767,0.11960533261299133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,32767,0.09643200039863586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,32767,0.08768533666928609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,32767,0.08676266670227051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,32767,0.08687466382980347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,32767,0.09444800019264221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,32767,0.09547199805577596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,32767,0.11967466274897258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,32767,0.09210667014122009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,32767,0.08698667089144389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,32767,0.08686400453249614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,32767,0.08702400326728821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,65535,0.14991999665896097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,65535,0.14654399951299033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,65535,0.13962133725484213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,65535,0.19866132736206055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,65535,0.134661336739858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,65535,0.1367946664492289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,65535,0.1339359978834788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,65535,0.15025066335995993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,65535,0.14453333616256714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,65535,0.14100799957911173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,65535,0.13617066542307535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,65535,0.13538133104642233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,65535,0.1349440018335978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,1,131071,0.2576853235562642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,65535,0.19777067502339682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,2,131071,0.24067733685175577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,4,131071,0.23527467250823975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,8,131071,0.2347573240598043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,16,131071,0.23099199930826822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,32,131071,0.2305813431739807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,4,1,64,131071,0.22985066970189413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,2,131071,0.2405386765797933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,1,131071,0.36562132835388184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,4,131071,0.23455999294916788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,16,131071,0.2328480084737142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,8,131071,0.2316853404045105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,32,131071,0.2307413419087728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,1,0.023685333629449207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,1,0.02882133424282074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,1,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,4,1,64,131071,0.22976533571879068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,1,0.02759466568628947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,1,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,1,0.027215999861558277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,1,0.027258666853109997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,1,0.023872000475724537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,1,0.028202667832374573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,1,0.02871999889612198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,1,0.027610667049884796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,1,0.02714666724205017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,1,0.02734400083621343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,1,0.0271519993742307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,3,0.024266667664051056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,3,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,3,0.02923733244339625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,3,0.027845333019892376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,3,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,3,0.027221334477265675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,3,0.027290667096773785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,3,0.02386133372783661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,3,0.028858666618665058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,3,0.027776000400384266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,3,0.027136000494162243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,7,0.023984000086784363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,3,0.027119999130566914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,3,0.02792533238728841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,3,0.027056001126766205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,7,0.02846933404604594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,7,0.027610667049884796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,7,0.02741333345572154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,7,0.027119999130566914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,7,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,7,0.027045334378878277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,7,0.0235359991590182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,7,0.028853334486484528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,7,0.02775466690460841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,7,0.027189334233601887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,7,0.027269333600997925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,7,0.027077332139015198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,7,0.027263998985290527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,15,0.027514666318893433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,15,0.02868266652027766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,15,0.023951999843120575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,15,0.027130665878454845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,15,0.02683199942111969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,15,0.026528000831604004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,15,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,15,0.0239680012067159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,15,0.028437333802382152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,15,0.0276853342851003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,15,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,15,0.02679466704527537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,15,0.026608000199000042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,31,0.023002666731675465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,15,0.026704000929991405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,31,0.027999999622503918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,31,0.026736001173655193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,31,0.026928000152111053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,31,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,31,0.02613866577545802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,31,0.023120000958442688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,31,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,31,0.027866666515668232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,31,0.027087998886903126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,31,0.026394667724768322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,31,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,31,0.02643200010061264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,63,0.022277332842350006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,31,0.026170666019121807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,63,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,63,0.025973332424958546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,63,0.02584533393383026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,63,0.025077333052953083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,63,0.025573333104451496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,63,0.025061334172884624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,63,0.022319999833901722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,63,0.0268053337931633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,63,0.025850666066010792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,63,0.025493333737055462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,63,0.025093334416548412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,63,0.025072000920772552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,63,0.025536000728607178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,127,0.026373334228992462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,127,0.030165334542592365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,127,0.028853334486484528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,127,0.02829333394765854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,127,0.028234665592511494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,127,0.027664000789324444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,127,0.02770666778087616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,127,0.0262719988822937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,127,0.028815999627113342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,127,0.02852799991766612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,127,0.030159999926884968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,127,0.028058665494124096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,127,0.02777066578467687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,127,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,255,0.030613332986831665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,255,0.02848000079393387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,255,0.030037333567937214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,255,0.032730666299661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,255,0.029578665892283123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,255,0.029130667448043823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,255,0.028858666618665058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,255,0.028368001182874043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,255,0.032405334214369454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,255,0.02997333308060964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,255,0.031040000418821972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,255,0.029616000751654308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,255,0.029077333708604176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,255,0.02900800108909607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,511,0.03027733415365219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,511,0.03292266776164373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,511,0.035743998984495796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,511,0.03219199925661087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,511,0.03197333216667175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,511,0.0314026673634847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,511,0.03141333411137263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,511,0.035743998984495796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,511,0.029839999973773956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,511,0.0330079992612203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,511,0.03216533362865448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,511,0.03181333343187968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,511,0.03148266673088074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,511,0.031498665610949196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,1023,0.03282133241494497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,1023,0.03498133271932602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,1023,0.03523733218510946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,1023,0.037989333271980286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,1023,0.03417599946260452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,1023,0.03401600072781245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,1023,0.03421333432197571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,1023,0.03342933456103007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,1023,0.03826666623353958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,1023,0.03579733272393545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,1023,0.034501334031422935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,1023,0.03444266567627589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,1023,0.03416533271471659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,1023,0.03396799912055334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,2047,0.03846399982770284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,2047,0.04167466859022776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,2047,0.03977599988381068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,2047,0.03920000046491623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,2047,0.03827200084924698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,2047,0.038719999293486275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,2047,0.03789333254098892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,2047,0.039488000174363456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,2047,0.041893333196640015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,2047,0.03951466580231985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,2047,0.0388373335202535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,2047,0.03888533264398575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,2047,0.03832533210515976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,2047,0.03826133410135905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,4095,0.0631466656923294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,4095,0.054645334680875145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,4095,0.050986667474110924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,4095,0.044165333112080894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,4095,0.04520000020662943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,4095,0.044480000933011375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,4095,0.04256533086299896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,4095,0.051738664507865906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,4095,0.06346666812896729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,4095,0.05134400228659312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,4095,0.04534400006135305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,4095,0.04478933413823446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,4095,0.04267199834187826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,4095,0.042863999803860985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,8191,0.06858133276303609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,8191,0.0711359977722168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,8191,0.06520000100135803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,8191,0.08397333820660909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,8191,0.06229333579540253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,8191,0.06357333560784657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,8191,0.06233599781990051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,8191,0.06809600194295247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,8191,0.06448533137639363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,8191,0.0713866651058197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,8191,0.07974933087825775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,8191,0.06649066507816315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,8191,0.06234666705131531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,8191,0.06289066871007283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,16383,0.09453333417574565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,16383,0.09690133730570476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,16383,0.09010133147239685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,16383,0.08751466870307922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,16383,0.11785067121187846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,16383,0.08530132969220479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,16383,0.08573333422342937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,16383,0.09482133388519287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,16383,0.0962666670481364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,16383,0.09032533566157024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,16383,0.08742400010426839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,16383,0.08664000034332275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,16383,0.11674132943153381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,16383,0.08555733164151509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,32767,0.14948266744613647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,32767,0.14481066664059958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,32767,0.13766933480898538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,32767,0.13879467050234476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,32767,0.13475199540456137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,32767,0.13367467125256857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,32767,0.19450666507085165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,32767,0.15043200055758157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,32767,0.1462506651878357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,32767,0.13884799679120383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,32767,0.13826133807500204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,32767,0.13517866532007852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,32767,0.13411733508110046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,32767,0.13344533244768778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,65535,0.3678773244222005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,65535,0.24050132433573404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,65535,0.23658132553100586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,65535,0.23093332846959433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,65535,0.2307200034459432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,65535,0.22919466098149618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,65535,0.22897599140803018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,65535,0.25914132595062256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,65535,0.3671199878056844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,65535,0.23334934314092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,65535,0.23341333866119385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,65535,0.23034665981928507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,65535,0.22883733113606772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,65535,0.2297439972559611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,1,131071,0.47464533646901447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,2,131071,0.43009599049886066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,8,131071,0.42398401101430255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,16,131071,0.4169600009918213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,4,131071,0.6700320243835449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,32,131071,0.4169173240661621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,8,1,64,131071,0.41633601983388263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,2,131071,0.43053332964579266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,1,131071,0.4738453229268392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,8,131071,0.42341868082682294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,4,131071,0.4224426746368408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,16,131071,0.4172746737798055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,32,131071,0.6627519925435384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,8,1,64,131071,0.4177066485087077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,1,0.03014933317899704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,1,0.026954665780067444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,1,0.029605334003766377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,1,0.028565332293510437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,1,0.028250666956106823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,1,0.02827200045188268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,1,0.02827200045188268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,1,0.030234667162100475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,1,0.026928000152111053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,1,0.029285334050655365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,1,0.028880000114440918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,1,0.028346667687098186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,1,0.028250666956106823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,1,0.028192001084486645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,3,0.02681066592534383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,3,0.03009066730737686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,3,0.02886933336655299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,3,0.028698667883872986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,3,0.028437333802382152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,3,0.028160000840822857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,3,0.028058665494124096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,3,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,3,0.030213333666324615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,3,0.028965334097544353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,3,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,3,0.028346667687098186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,3,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,3,0.028138667345046997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,7,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,7,0.02905600021282832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,7,0.029818666477998097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,7,0.028336000939210255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,7,0.027989332874615986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,7,0.027888000011444092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,7,0.028389332195123036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,7,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,7,0.029978667696317036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,7,0.028736000259717304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,7,0.028143999477227528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,7,0.02826133370399475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,7,0.027914665639400482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,15,0.029616000751654308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,7,0.028016000986099243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,15,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,15,0.028549333413441975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,15,0.027855999767780304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,15,0.027808000644048054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,15,0.02755733331044515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,15,0.027727998793125153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,15,0.025941332181294758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,15,0.029733332494894665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,15,0.028832000990708668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,15,0.02787200113137563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,15,0.02779199928045273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,15,0.02759466568628947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,15,0.027482666075229645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,31,0.028944000601768494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,31,0.025946666797002155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,31,0.02812800059715907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,31,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,31,0.02714666724205017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,31,0.027119999130566914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,31,0.027263998985290527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,31,0.025754667818546295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,31,0.028965334097544353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,31,0.02852799991766612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,31,0.027722666660944622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,31,0.027562665442625683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,31,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,31,0.02703999976317088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,63,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,63,0.028117333849271137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,63,0.02703999976317088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,63,0.026474667092164356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,63,0.026314665873845417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,63,0.02630399912595749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,63,0.026074667771657307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,63,0.025125332176685333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,63,0.027077332139015198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,63,0.028250666956106823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,63,0.026602665583292644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,63,0.0259253333012263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,63,0.02626666675011317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,63,0.026335999369621277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,127,0.03338133295377096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,127,0.026890667776266735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,127,0.03031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,127,0.030234667162100475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,127,0.031002665559450786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,127,0.029722665747006733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,127,0.02979733298222224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,127,0.02630399912595749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,127,0.033471999069054924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,127,0.031173333525657654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,127,0.03025600065787633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,127,0.03030933439731598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,127,0.029909332593282063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,127,0.02978666623433431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,255,0.032842665910720825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,255,0.033973333736260734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,255,0.03565866748491923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,255,0.03288000077009201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,255,0.032885332902272545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,255,0.03230933348337809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,255,0.0329120010137558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,255,0.031514666974544525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,255,0.035743998984495796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,255,0.03395200024048487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,255,0.03290133426586787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,255,0.0327360009153684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,255,0.03234666585922241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,255,0.03206400076548258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,511,0.03568000098069509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,511,0.03827200084924698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,511,0.03509333233038584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,511,0.037104000647862755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,511,0.03419733295838038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,511,0.034586665530999504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,511,0.034234667817751564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,511,0.03541333228349686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,511,0.04020266731580099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,511,0.03586666782697042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,511,0.034874667723973594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,511,0.035418666899204254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,511,0.034128000338872276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,511,0.03400533397992452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,1023,0.04099733382463455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,1023,0.044112001856168113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,1023,0.04213866591453552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,1023,0.03931200007597605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,1023,0.04027199993530909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,1023,0.03912533322970072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,1023,0.038959999879201256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,1023,0.04378133515516917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,1023,0.04240000247955322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,1023,0.041589332123597465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,1023,0.041077333192030586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,1023,0.03905066599448522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,1023,0.03922666609287262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,1023,0.03897066662708918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,2047,0.056320001681645714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,2047,0.05474133292833964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,2047,0.050944000482559204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,2047,0.05564799904823303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,2047,0.04506133496761322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,2047,0.045466666420300804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,2047,0.04393066465854645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,2047,0.055434669057528176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,2047,0.056159997979799904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,2047,0.05197866757710775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,2047,0.04900266726811727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,2047,0.04819199939568838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,2047,0.04585599899291992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,2047,0.04473066826661428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,4095,0.06919466455777486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,4095,0.06699733436107635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,4095,0.07245866457621257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,4095,0.06432533264160156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,4095,0.06444799900054932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,4095,0.0790880024433136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,4095,0.06234666705131531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,4095,0.06948266426722209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,4095,0.07241599758466084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,4095,0.0668639987707138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,4095,0.06470400094985962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,4095,0.06404800216356914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,4095,0.06396799782911937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,4095,0.07806933422883351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,8191,0.09804266691207886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,8191,0.10028266906738281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,8191,0.0897759993871053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,8191,0.09329600135485332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,8191,0.08829866846402486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,8191,0.08721599976221721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,8191,0.08801600337028503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,8191,0.10040000081062317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,8191,0.12659733494122824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,8191,0.09251200159390767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,8191,0.08927466471989949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,8191,0.08888000249862671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,8191,0.08716266353925069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,8191,0.0867680013179779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,16383,0.15099733074506125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,16383,0.14532267053922018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,16383,0.2013333241144816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,16383,0.1360373298327128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,16383,0.13569600383440653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,16383,0.13339733084042868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,16383,0.1333440045515696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,16383,0.15159466862678528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,16383,0.14702399571736655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,16383,0.13964800039927164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,16383,0.19822933276494345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,16383,0.13489066561063132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,16383,0.1346399982770284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,16383,0.1334933340549469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,32767,0.2641706665356954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,32767,0.24173333247502646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,32767,0.23643734057744345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,32767,0.35283732414245605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,32767,0.23229867219924927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,32767,0.23064533869425455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,32767,0.22852800289789835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,32767,0.2635466655095418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,32767,0.2424160043398539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,32767,0.23517866929372153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,32767,0.23322665691375732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,32767,0.22898133595784506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,32767,0.3543306589126587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,32767,0.22884267568588257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,65535,0.4794613520304362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,65535,0.4307253360748291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,65535,0.4240373373031616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,65535,0.4192426602045695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,65535,0.41840533415476483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,65535,0.4204853375752767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,65535,0.6623520056406657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,65535,0.480405330657959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,65535,0.4320746660232544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,65535,0.42394665877024335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,65535,0.4198293288548787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,65535,0.41886401176452637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,65535,0.41925867398579914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,65535,0.41644267241160077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,1,131071,1.312618652979533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,2,131071,0.8079946835835775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,4,131071,0.798746665318807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,8,131071,0.7952373027801514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,64,131071,0.7999413013458252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,16,131071,0.7932693163553873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,16,1,32,131071,0.7911093235015869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,1,131071,0.9095946947733561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,2,131071,1.3064213593800862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,4,131071,0.7998613516489664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,8,131071,0.7957279682159424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,16,131071,0.792901357014974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,1,0.030133334298928578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,32,131071,0.7987786928812662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,16,1,64,131071,0.7902080217997233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,1,0.03402133285999298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,1,0.031856000423431396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,1,0.030906667311986286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,1,0.030554667115211487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,1,0.030165334542592365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,1,0.030133334298928578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,1,0.029904000461101532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,1,0.03243733445803324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,1,0.03365866591533025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,1,0.03102933367093404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,1,0.03029866764942805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,1,0.030618667602539062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,1,0.03012266755104065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,3,0.033370666205883026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,3,0.02996266633272171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,3,0.03070933371782303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,3,0.032511999209721885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,3,0.03049066662788391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,3,0.030378667016824085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,3,0.030586667358875275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,3,0.03329599897066752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,3,0.031680000325044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,3,0.029877332349618275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,3,0.030837332208951313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,3,0.030767999589443207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,3,0.03044266750415166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,3,0.030415999392668407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,7,0.0332640012105306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,7,0.030042665700117748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,7,0.03166399896144867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,7,0.03054933249950409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,7,0.03048533449570338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,7,0.03031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,7,0.030245333909988403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,7,0.029898665845394135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,7,0.03320533285538355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,7,0.0313226655125618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,7,0.030752000709374745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,7,0.030282666285832722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,7,0.030394665896892548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,7,0.030447999636332195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,15,0.029525332152843475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,15,0.03289066751797994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,15,0.030954666435718536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,15,0.030207999050617218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,15,0.02985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,15,0.029701332251230877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,15,0.029887999097506206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,15,0.02902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,15,0.033002667129039764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,15,0.031093334158261616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,15,0.029919999341169994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,15,0.029824001093705494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,15,0.02980799973011017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,15,0.0296426663796107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,31,0.030789333085219067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,31,0.02889599899450938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,31,0.03305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,31,0.029520000020662945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,31,0.029093332588672638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,31,0.029365333418051403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,31,0.029343999922275543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,31,0.028688001135985058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,31,0.03266666581233343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,31,0.030981334547201794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,31,0.030453334252039593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,31,0.02917333443959554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,31,0.029098667204380035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,31,0.02942399928967158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,63,0.028213332096735638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,63,0.032058666149775185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,63,0.03013866643110911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,63,0.029109333952267964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,63,0.028575999041398365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,63,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,63,0.02834133307139079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,63,0.028624000648657482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,63,0.032127998769283295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,63,0.02977599948644638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,63,0.028949332733949024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,63,0.028368001182874043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,63,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,63,0.02829333394765854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,127,0.029994666576385498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,127,0.03504000107447306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,127,0.032399999598662056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,127,0.031541332602500916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,127,0.03105599929889043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,127,0.031125334401925404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,127,0.031178665657838184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,127,0.03473066786924998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,127,0.030368000268936157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,127,0.03249066571394602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,127,0.03164266546567281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,127,0.03127466638882955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,127,0.031040000418821972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,127,0.031173333525657654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,255,0.04164800047874451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,255,0.0332640012105306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,255,0.03826133410135905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,255,0.037392000357309975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,255,0.03665599972009659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,255,0.037050666908423104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,255,0.03707200040419897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,255,0.041797334949175514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,255,0.03397866586844126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,255,0.037765334049860634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,255,0.03740799923737844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,255,0.037087999284267426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,255,0.03703466554482778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,255,0.036501333117485046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,511,0.04943466683228811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,511,0.0452159990866979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,511,0.04238399863243103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,511,0.039349332451820374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,511,0.04021333406368891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,511,0.038912000755469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,511,0.04001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,511,0.04964800179004669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,511,0.04489600161711375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,511,0.04079466561476389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,511,0.04083733260631561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,511,0.03924266745646795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,511,0.03940266619126002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,511,0.0390079990029335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,1023,0.06341866652170818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,1023,0.05216533442338308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,1023,0.05769066512584686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,1023,0.048565333088239036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,1023,0.0470719983180364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,1023,0.045968001087506614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,1023,0.0451200008392334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,1023,0.06308799982070923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,1023,0.05846933523813883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,1023,0.05172266562779745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,1023,0.04675200084845225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,1023,0.04994666576385498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,1023,0.04682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,1023,0.04492799937725067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,2047,0.09430932998657227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,2047,0.07426133255163829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,2047,0.06842133402824402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,2047,0.06363733112812042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,2047,0.0655626654624939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,2047,0.0633493314186732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,2047,0.06298133234182994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,2047,0.07579733431339264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,2047,0.09325866897900899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,2047,0.06807466844717662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,2047,0.06560533245404561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,2047,0.06381333371003468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,2047,0.0631520003080368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,2047,0.06298666695753734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,4095,0.08928533395131429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,4095,0.10430399576822917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,4095,0.12544000148773193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,4095,0.09990933537483215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,4095,0.08755200107892354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,4095,0.0867733359336853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,4095,0.10391466816266377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,4095,0.0865280032157898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,4095,0.09885866443316142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,4095,0.09269866347312927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,4095,0.12086400389671326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,4095,0.08850133419036865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,4095,0.08610666791598003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,4095,0.08738666772842407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,8191,0.15781866510709128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,8191,0.14698666334152222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,8191,0.14006400108337402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,8191,0.13716800014177957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,8191,0.19793067375818887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,8191,0.13596266508102417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,8191,0.1329973340034485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,8191,0.15812266866366068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,8191,0.14703999956448874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,8191,0.13979199528694153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,8191,0.1369439959526062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,8191,0.1358453333377838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,8191,0.19728533426920572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,8191,0.13497599959373474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,16383,0.2733706633249919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,16383,0.24473599592844644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,16383,0.23531200488408408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,16383,0.23193599780400595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,16383,0.23061333100001016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,16383,0.3502346674601237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,16383,0.22979199886322021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,16383,0.27268266677856445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,16383,0.243504007657369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,16383,0.23591466744740805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,16383,0.23205333948135376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,16383,0.23030932744344076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,16383,0.22959999243418375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,16383,0.35016000270843506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,32767,0.48923198382059735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,32767,0.43113601207733154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,32767,0.4403040011723836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,32767,0.4223466714223226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,32767,0.41996800899505615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,32767,0.4190133412679036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,32767,0.4174293279647827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,32767,0.7044373353322347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,32767,0.43226667245229083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,32767,0.4397653341293335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,32767,0.42451731363932294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,32767,0.4196586608886719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,32767,0.41767998536427814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,32767,0.4182613293329875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,65535,0.9206613699595133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,65535,0.8105759620666504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,65535,1.3158559799194336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,65535,0.7968479792277018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,65535,0.7946879863739014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,65535,0.7926026980082194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,65535,0.7912320295969645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,65535,0.9202240308125814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,65535,0.8155306975046793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,65535,0.8107253710428873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,65535,1.289669354756673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,65535,0.7936320304870605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,65535,0.793615976969401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,65535,0.7914986610412598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,4,131071,1.5493440628051758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,2,131071,1.5630240440368652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,1,131071,1.7820587158203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,8,131071,1.5630772908528645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,16,131071,2.52675199508667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,64,131071,1.5383787155151367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,32,1,32,131071,1.5403092702229817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,1,131071,1.7831573486328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,2,131071,1.5651520093282063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,4,131071,1.5514453252156575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,1,0.036501333117485046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,16,131071,1.5580639839172363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,1,0.038805333276589714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,32,131071,2.5222293535868325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,64,131071,1.5414613087972004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,32,1,8,131071,1.5455519358317058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,1,0.03656533360481262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,1,0.035232000052928925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,1,0.03492266684770584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,1,0.03433600068092346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,1,0.03470933437347412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,1,0.037061333656311035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,1,0.03930133332808813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,1,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,1,0.036303999523321785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,1,0.034373333056767784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,1,0.03480000048875809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,1,0.034586665530999504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,3,0.03696533292531967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,3,0.03879466652870178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,3,0.03503466645876566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,3,0.036271999279658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,3,0.03451200077931086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,3,0.03440000116825104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,3,0.03495466709136963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,3,0.03676266719897588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,3,0.03892799963553747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,3,0.03513066718975703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,3,0.03465066601832708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,3,0.034416000048319496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,3,0.03618133316437403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,3,0.03443733354409536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,7,0.036373332142829895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,7,0.038389332592487335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,7,0.03625600039958954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,7,0.0349386657277743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,7,0.034527999659379326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,7,0.03397866586844126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,7,0.03422933320204417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,7,0.03694933404525121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,7,0.039221333960692085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,7,0.03518400092919668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,7,0.03602666656176249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,7,0.034272000193595886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,7,0.033941333492596946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,7,0.03442133218050003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,15,0.03808533400297165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,15,0.0365226666132609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,15,0.03621333340803782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,15,0.03480533262093862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,15,0.03400533397992452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,15,0.03379199902216593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,15,0.033546666304270424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,15,0.0366239994764328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,15,0.03802666564782461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,15,0.03570133447647095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,15,0.034789333740870156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,15,0.0339626669883728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,15,0.03390933324893316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,15,0.033941333492596946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,31,0.035002666215101876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,31,0.035989334185918175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,31,0.03809066613515218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,31,0.03393599887688955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,31,0.03379733363787333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,31,0.0334346666932106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,31,0.03340800106525421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,31,0.03615466753641764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,31,0.037418665985266365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,31,0.035317334036032356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,31,0.03418133407831192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,31,0.03350399931271871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,31,0.03345600018898646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,31,0.03367999941110611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,63,0.037946666280428566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,63,0.03506666670242945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,63,0.034490667283535004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,63,0.03281066566705704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,63,0.03331200033426285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,63,0.03265066693226496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,63,0.037861332297325134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,63,0.03236799935499827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,63,0.034645333886146545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,63,0.035216001172860466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,63,0.0332640012105306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,63,0.03239466746648153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,63,0.03249600032965342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,63,0.03224000086386999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,127,0.038015998899936676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,127,0.040735999743143715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,127,0.03737066686153412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,127,0.03675200045108795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,127,0.03545066714286804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,127,0.03502399971087774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,127,0.035562666753927864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,127,0.03885333240032196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,127,0.04011733333269755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,127,0.037658666570981346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,127,0.036117332677046456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,127,0.03608000030120214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,127,0.03521066655715307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,127,0.03501333296298981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,255,0.04375466704368591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,255,0.04364799956480662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,255,0.04095466683308283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,255,0.04026666780312856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,255,0.038773333032925926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,255,0.03994666785001755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,255,0.03890133400758108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,255,0.04318933188915253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,255,0.043935999274253845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,255,0.03995199998219808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,255,0.0407679999868075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,255,0.03892799963553747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,255,0.039690665900707245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,255,0.03990400085846583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,511,0.05571199953556061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,511,0.06572799881299336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,511,0.05198933184146881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,511,0.06692799925804138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,511,0.0505920002857844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,511,0.048538664976755776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,511,0.04930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,511,0.05416533350944519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,511,0.06553600231806438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,511,0.05190399785836538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,511,0.05384533107280731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,511,0.058543999989827476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,511,0.04930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,511,0.049509331583976746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,1023,0.07121066749095917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,1023,0.0848426620165507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,1023,0.07414933542410533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,1023,0.0692853331565857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,1023,0.08319466809431712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,1023,0.06671466430028279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,1023,0.0659093310435613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,1023,0.06941866874694824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,1023,0.08496000369389851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,1023,0.07480533421039581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,1023,0.06870933373769124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,1023,0.06726400057474773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,1023,0.08210133512814839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,1023,0.06659199794133504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,2047,0.09795199831326802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,2047,0.0972213347752889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,2047,0.10684266686439514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,2047,0.09404800335566203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,2047,0.09199999769528706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,2047,0.09056533376375835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,2047,0.11997866630554199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,2047,0.10682666301727295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,2047,0.0981280008951823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,2047,0.09729066491127014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,2047,0.09307733178138733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,2047,0.09239466985066731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,2047,0.09108266234397888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,2047,0.09072533249855042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,4095,0.20622400442759195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,4095,0.1532373329003652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,4095,0.14598400394121805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,4095,0.1406826674938202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,4095,0.1397546629110972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,4095,0.13801599542299905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,4095,0.15362667044003805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,4095,0.13954133788744608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,4095,0.1439520021279653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,4095,0.21953066190083823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,4095,0.14083733161290488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,4095,0.13933866222699484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,4095,0.1395093301932017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,4095,0.13795733451843262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,8191,0.26582932472229004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,8191,0.25064533948898315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,8191,0.2393653392791748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,8191,0.358298659324646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,8191,0.23386667172114053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,8191,0.23310933510462442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,8191,0.23335999250411987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,8191,0.2510506709416707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,8191,0.26450665791829425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,8191,0.24013332525889078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,8191,0.2323466738065084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,8191,0.3556906779607137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,8191,0.23386132717132568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,8191,0.2326293389002482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,16383,0.45475200812021893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,16383,0.5083946784337362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,16383,0.4380799929300944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,16383,0.42665600776672363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,16383,0.4283359845479329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,16383,0.6677493254343668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,16383,0.42132266362508136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,16383,0.5090986490249634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,16383,0.45398398240407306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,16383,0.4404960076014201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,16383,0.42863468329111737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,16383,0.4245226780573527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,16383,0.42242133617401123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,16383,0.6666400035222372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,32767,0.9398240248362223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,32767,0.8125653266906738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,32767,0.8301653067270914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,32767,0.8042346636454264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,32767,0.8035199642181396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,32767,0.7986186345418295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,32767,0.7958079973856608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,32767,1.3651466369628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,32767,0.8300373554229736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,32767,0.8127520084381104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,32767,0.8029759724934896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,32767,0.8069547017415365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,32767,0.7984960079193115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,32767,0.7945013046264648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,65535,2.5765493710835776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,65535,1.560490608215332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,65535,1.545610745747884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,65535,1.5602933565775554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,65535,1.5501279830932617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,65535,1.5447413126627605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,65535,1.799359957377116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,65535,1.7985599835713704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,65535,2.573967933654785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,65535,1.5622986157735188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,65535,1.5507094065348308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,65535,1.5656639734903972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,65535,1.5505493481953938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,65535,1.5430186589558919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,2,131071,3.0749759674072266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,1,131071,3.5201441446940103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,8,131071,3.0434347788492837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,16,131071,3.0390345255533853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,4,131071,5.027418772379558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,64,131071,3.0403518676757812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,1,131071,3.519514719645182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,64,1,32,131071,3.0720532735188804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,2,131071,3.0769707361857095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,1,0.056320001681645714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,16,131071,3.0406080881754556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,1,0.0489279975493749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,8,131071,3.0452054341634116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,32,131071,5.002127965291341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,64,131071,3.0355679194132485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,1,0.04497066636880239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,1,0.04299733539422353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,64,1,4,131071,3.0562238693237305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,1,0.041946664452552795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,1,0.04164266586303711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,1,0.041349334021409355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,1,0.05590933561325073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,1,0.045194665590922035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,1,0.043040002385775246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,1,0.04911466439565023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,1,0.04145599901676178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,1,0.04174399872620901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,1,0.04151466737190882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,3,0.048783997694651283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,3,0.05522133409976959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,3,0.04518933097521464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,3,0.042768001556396484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,3,0.04207466542720795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,3,0.041573333243529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,3,0.05524266759554545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,3,0.041306667029857635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,3,0.04866666595141093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,3,0.04310933252175649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,3,0.044938668608665466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,3,0.041434665520985924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,3,0.04173333446184794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,3,0.04201066493988037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,7,0.055674667159716286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,7,0.044495999813079834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,7,0.04173333446184794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,7,0.04311466713746389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,7,0.04863466819127401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,7,0.04140799989302953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,7,0.04140799989302953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,7,0.04887466629346212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,7,0.04260799785455068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,7,0.04452266792456309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,7,0.04214400053024292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,7,0.05795733133951823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,7,0.04132800052563349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,7,0.04131199916203817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,15,0.055349335074424744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,15,0.044154668847719826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,15,0.04853333532810211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,15,0.04200000067551931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,15,0.041663999358812966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,15,0.041093334555625916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,15,0.04089066634575526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,15,0.05552533268928528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,15,0.04836266736189524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,15,0.04436799883842468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,15,0.042410666743914284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,15,0.04115733255942663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,15,0.041077333192030586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,15,0.041077333192030586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,31,0.06376533210277557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,31,0.04390400151411692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,31,0.04156800111134847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,31,0.047925333182017006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,31,0.041093334555625916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,31,0.04088533421357473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,31,0.04030933231115341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,31,0.05739733576774597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,31,0.04146666576464971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,31,0.04782933493455251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,31,0.04387733340263367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,31,0.04044266790151596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,31,0.04104000081618627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,31,0.04074133435885111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,63,0.05938666562239329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,63,0.04833599925041199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,63,0.04108799993991852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,63,0.04326933125654856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,63,0.040133332212766014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,63,0.0396373321612676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,63,0.039279999832312264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,63,0.0433599998553594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,63,0.047983999053637184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,63,0.059664001067479454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,63,0.04107200105985006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,63,0.03975466638803482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,63,0.0401706670721372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,63,0.03951466580231985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,127,0.06385066608587901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,127,0.05269866685072581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,127,0.046341334780057274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,127,0.04404800136884054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,127,0.04342933495839437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,127,0.04298133154710134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,127,0.04273599882920583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,127,0.05228800078233083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,127,0.06489066779613495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,127,0.04637866715590159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,127,0.044122666120529175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,127,0.04278933505217234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,127,0.042863999803860985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,127,0.04318933188915253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,255,0.07276266813278198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,255,0.06562133133411407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,255,0.05222400029500326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,255,0.04804799954096476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,255,0.05705066521962484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,255,0.04756266872088114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,255,0.04762666424115499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,255,0.058703998724619545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,255,0.06381866832574208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,255,0.0528106689453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,255,0.07200533151626587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,255,0.04846400022506714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,255,0.0476746658484141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,255,0.04758933186531067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,511,0.08732799688975017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,511,0.07850666840871175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,511,0.07086933155854543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,511,0.06790933509667714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,511,0.0803466687599818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,511,0.06557866434256236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,511,0.0664160003264745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,511,0.0772159993648529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,511,0.07261866827805837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,511,0.08744532863299052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,511,0.06648000081380208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,511,0.06790933509667714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,511,0.08136533200740814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,511,0.06558399895826976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,1023,0.10116266210873921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,1023,0.09106133381525676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,1023,0.09624532858530681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,1023,0.09038399656613667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,1023,0.11622933546702068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,1023,0.08957866827646892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,1023,0.11927466591199239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,1023,0.1160640021165212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,1023,0.10066133737564087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,1023,0.09469333291053772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,1023,0.09005866448084514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,1023,0.09354133407274882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,1023,0.08970666925112407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,2047,0.22914665937423706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,1023,0.08940800031026204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,2047,0.15016000469525656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,2047,0.14356266458829245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,2047,0.13965333501497904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,2047,0.1398240029811859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,2047,0.1383946637312571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,2047,0.13753066460291544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,2047,0.22737600406010947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,2047,0.15096533298492432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,2047,0.14324266711870828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,2047,0.1397333343823751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,2047,0.1409440040588379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,2047,0.1365013321240743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,2047,0.13782933354377747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,4095,0.3692479928334554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,4095,0.2824746568997701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,4095,0.23881065845489502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,4095,0.23397332429885864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,4095,0.23504533370335898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,4095,0.23473066091537476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,4095,0.22961600621541342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,4095,0.2825653354326884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,4095,0.36927465597788495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,4095,0.23937066396077475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,4095,0.23358933130900064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,4095,0.23348800341288248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,4095,0.2326293389002482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,4095,0.2339306672414144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,8191,0.435535987218221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,8191,0.521343986193339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,8191,0.6717867056528727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,8191,0.42507731914520264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,8191,0.42282132307688397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,8191,0.42366401354471844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,8191,0.421674648920695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,8191,0.5192799965540568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,8191,0.4352693160374959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,8191,0.6686293284098307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,8191,0.43058133125305176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,8191,0.42215998967488605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,8191,0.42346131801605225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,8191,0.4224426746368408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,16383,0.9538133144378662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,16383,0.8601813316345215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,16383,0.8328266938527426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,16383,1.3106773694356282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,16383,0.8030133247375488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,16383,0.8006079991658529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,16383,0.8158506552378336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,16383,0.9546826680501302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,16383,0.8339946269989014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,16383,0.8578613599141439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,16383,1.3000160058339436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,16383,0.8154506683349609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,16383,0.8003733158111572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,16383,0.803167978922526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,32767,1.5861387252807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,32767,1.6108105977376301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,32767,1.5620320638020833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,32767,1.8163520495096843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,32767,1.5499839782714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,32767,1.547445297241211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,32767,2.5386239687601724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,32767,1.816149393717448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,32767,1.6095840136210124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,32767,1.5648105939229329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,32767,1.549221356709798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,32767,1.5819519360860188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,32767,1.5697973569234211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,32767,1.549280007680257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,65535,3.1077706019083657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,65535,3.0766026178995767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,65535,5.132026672363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,65535,3.054746627807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,65535,3.0758771896362305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,65535,3.0474878946940103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,65535,3.537328084309896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,65535,3.0420106252034507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,65535,3.1082932154337564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,65535,3.0772854487101235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,65535,3.04473082224528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,65535,3.071839968363444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,65535,3.0383679072062173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,65535,5.026533444722493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,2,131071,6.091946919759114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,1,131071,6.9729868570963545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,32,131071,6.096933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,64,131071,6.021338780721028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,16,131071,6.028037389119466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,8,131071,6.043962478637695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,128,1,4,131071,6.058922449747722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,1,131071,10.099344253540039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,1,0.09499733646710713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,1,0.07233066856861115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,1,0.06099733213583628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,1,0.05748266478379568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,2,131071,6.091002782185872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,1,0.05593066910902659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,8,131071,6.1123199462890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,16,131071,6.034069061279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,1,0.05535466472307841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,1,0.05495999753475189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,1,0.09478400150934856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,1,0.06108800073464712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,1,0.07228800157705943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,1,0.05745066702365875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,1,0.05596800148487091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,4,131071,6.064165115356445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,1,0.05516799787680308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,1,0.05509333312511444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,3,0.06092800199985504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,3,0.07043733199437459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,3,0.10615467031796773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,64,131071,6.0244903564453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,128,1,32,131071,6.027578353881836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,3,0.057402665416399636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,3,0.05605333546797434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,3,0.05528533458709717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,3,0.05467733244101206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,3,0.09408533573150635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,3,0.0728000005086263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,3,0.06107200185457865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,3,0.05613866448402405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,3,0.05779199798901876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,3,0.05477866530418396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,3,0.05471999943256378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,7,0.0937439997990926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,7,0.07037866612275441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,7,0.060677334666252136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,7,0.057061334451039634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,7,0.05553600192070007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,7,0.0544106662273407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,7,0.054986665646235146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,7,0.09392000238100688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,7,0.07036266724268596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,7,0.060559997955958046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,7,0.0572320024172465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,7,0.055589333176612854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,7,0.054773335655530296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,7,0.054485330979029335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,15,0.09212266405423482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,15,0.05994133154551188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,15,0.05611200133959452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,15,0.0713919997215271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,15,0.05473066866397858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,15,0.05397333204746246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,15,0.05356266597906748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,15,0.0943946639696757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,15,0.07221333185831706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,15,0.05969599882761637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,15,0.0562720000743866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,15,0.05454400181770325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,15,0.053818667928377785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,15,0.053455998500188194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,31,0.0952959954738617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,31,0.07365866502126057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,31,0.05532800157864889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,31,0.05374933282534281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,31,0.06007466713587443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,31,0.05259733398755392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,31,0.0525493323802948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,31,0.09514666597048442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,31,0.05533866584300995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,31,0.059258664647738137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,31,0.0537013312180837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,31,0.07383466760317485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,31,0.05240533252557119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,31,0.05358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,63,0.11400000254313152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,63,0.07739733159542084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,63,0.051967998345692955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,63,0.05189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,63,0.05978666742642721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,63,0.0540533314148585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,63,0.050981332858403526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,63,0.0962666670481364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,63,0.08840533097585042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,63,0.05395199855168661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,63,0.06151466568311056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,63,0.05226666728655497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,63,0.051514665285746254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,63,0.05093866586685181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,127,0.10726400216420491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,127,0.08547733227411906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,127,0.0830506682395935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,127,0.06603200236956279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,127,0.06165333092212677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,127,0.0584799995024999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,127,0.05706666906674703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,127,0.10763733585675557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,127,0.08334933718045552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,127,0.07520000139872234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,127,0.07365866502126057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,127,0.061679999033610024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,127,0.05735999842484792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,127,0.05704000095526377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,255,0.09693333506584167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,255,0.0846026639143626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,255,0.08144533137480418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,255,0.07550399998823802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,255,0.1197760005791982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,255,0.07478933533032735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,255,0.0922826627890269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,255,0.1200320025285085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,255,0.08529599507649739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,255,0.09633066256841023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,255,0.08142933249473572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,255,0.07694399853547414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,255,0.07482133309046428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,255,0.07451733450094859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,511,0.18877333402633667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,511,0.12306132912635803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,511,0.10413333773612976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,511,0.10243733723958333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,511,0.10982400178909302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,511,0.09771200021107991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,511,0.09966933727264404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,511,0.16011200348536173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,511,0.14844266573588052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,511,0.11038933197657268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,511,0.10460266470909119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,511,0.10223467151323955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,511,0.09854933619499207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,511,0.10003200173377991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,1023,0.2064746618270874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,1023,0.23707199096679688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,1023,0.15151466925938925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,1023,0.14825066924095154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,1023,0.1479520003000895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,1023,0.15785066286722818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,1023,0.14516266187032065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,1023,0.17148266235987344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,1023,0.20589866240819296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,1023,0.15095466375350952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,1023,0.22150399287541708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,1023,0.14964266618092856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,1023,0.1462453305721283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,1023,0.1458506683508555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,2047,0.26547733942667645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,2047,0.25169066588083905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,2047,0.24549333254496256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,2047,0.3165760040283203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,2047,0.24259734153747559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,2047,0.3649760087331136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,2047,0.240064005057017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,2047,0.31749866406122845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,2047,0.2656480073928833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,2047,0.2512320081392924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,2047,0.24597867329915366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,2047,0.2418773372968038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,2047,0.36158398787180585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,2047,0.2421600023905436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,4095,0.5496319929758707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,4095,0.4540746609369914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,4095,0.44174933433532715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,4095,0.4358186721801758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,4095,0.4302826722462972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,4095,0.4316800038019816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,4095,0.6717493534088135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,4095,0.44122131665547687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,4095,0.45422931512196857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,4095,0.43597865104675293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,4095,0.43197866280873615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,4095,0.5499360164006551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,4095,0.4289066791534424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,4095,0.43034132321675617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,8191,1.432037353515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,8191,0.845952033996582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,8191,0.8289279937744141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,8191,0.8822933038075765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,8191,0.8141226768493652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,8191,0.8103840351104736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,8191,0.8110559781392416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,8191,1.388543923695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,8191,0.8295679887135824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,8191,0.8235092957814535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,8191,0.8427893320719401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,8191,0.8086667060852051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,8191,0.9847946961720785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,8191,0.8090293407440186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,16383,1.8485760688781738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,16383,1.5641172726949055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,16383,1.602842648824056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,16383,1.5914880434672039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,16383,2.644261360168457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,16383,1.556506633758545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,16383,1.8493226369222004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,16383,1.5586880048116047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,16383,1.6354559262593586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,16383,1.6009759902954102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,16383,1.58243195215861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,16383,1.5790665944417317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,16383,1.5576640764872234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,16383,2.540325323740641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,32767,3.5693868001302085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,32767,3.1376587549845376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,32767,3.0971787770589194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,32767,3.0748745600382485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,32767,3.0937067667643228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,32767,3.050800005594889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,32767,3.0504748026529946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,32767,5.186117490132649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,32767,3.1002400716145835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,32767,3.1375414530436196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,32767,3.07700252532959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,32767,3.056367874145508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,32767,3.0862401326497397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,32767,3.0523198445638022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,4,65535,6.084277470906575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,1,65535,7.003407796223958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,8,65535,6.065829594930013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,16,65535,6.040704091389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,2,65535,10.08505630493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,32,65535,6.034634908040364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,256,1,64,65535,6.110240300496419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,1,65535,7.006469090779622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,1,0.16326933105786642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,1,0.11460799972216289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,1,0.09571199615796407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,1,0.08691199620564778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,1,0.08383466800053914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,1,0.08213333288828532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,1,0.08137066662311554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,1,0.163290669520696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,1,0.1148533324400584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,1,0.0958026647567749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,1,0.08702400326728821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,8,65535,6.057562510172526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,1,0.08341866731643677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,64,65535,6.032229105631511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,16,65535,6.042298634847005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,1,0.0819893330335617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,4,65535,6.083840052286784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,2,65535,6.1292374928792315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,1,0.08130666613578796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,256,1,32,65535,9.960458755493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,3,0.11397332946459453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,3,0.09567466378211975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,3,0.08628799517949422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,3,0.08329066634178162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,3,0.16130133469899496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,3,0.08184533317883809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,3,0.08188266555468242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,3,0.160970667997996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,3,0.0862613320350647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,3,0.08339732885360718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,3,0.11428800225257874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,3,0.08186133205890656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,3,0.08105066418647766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,3,0.09770666559537251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,7,0.08636800448099773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,7,0.08255466818809509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,7,0.09556800127029419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,7,0.08116800089677174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,7,0.15848533312479654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,7,0.11336533228556316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,7,0.08038933575153351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,7,0.18764267365137735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,7,0.08166400094827016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,7,0.09538666407267253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,7,0.08057599763075511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,7,0.11398933331171672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,7,0.08643200000127156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,7,0.08269333342711131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,15,0.15933866302172342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,15,0.09648000200589497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,15,0.0848479966322581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,15,0.08084799846013387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,15,0.07944533228874207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,15,0.12958932916323343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,15,0.0784853349129359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,15,0.11397866408030193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,15,0.08074133098125458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,15,0.07923733194669087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,15,0.09633599718411763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,15,0.07875733574231465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,15,0.16053332885106406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,15,0.0849226713180542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,31,0.0827466646830241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,31,0.1162453293800354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,31,0.08001066744327545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,31,0.0956160028775533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,31,0.07749866445859273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,31,0.07708266874154408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,31,0.16157866517702738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,31,0.16181866327921549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,31,0.07918933530648549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,31,0.08578667044639587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,31,0.07760533193747203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,31,0.09488532940546672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,31,0.07693866888682048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,31,0.11556800206502278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,63,0.16686399777730307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,63,0.12101333340009053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,63,0.09981333216031392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,63,0.0919040044148763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,63,0.09091732899347942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,63,0.07915733257929485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,63,0.07619200150171916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,63,0.10017599662144978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,63,0.0886240005493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,63,0.08517866333325703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,63,0.12057066957155864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,63,0.07566399872303009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,63,0.0869599978129069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,63,0.16683199008305868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,127,0.18567999203999838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,127,0.09532266855239868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,127,0.09272533655166626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,127,0.10338667035102844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,127,0.11179733276367188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,127,0.10473066568374634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,127,0.13405332962671915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,127,0.09844266374905904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,127,0.11284266908963521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,127,0.09265066186587016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,127,0.13327999909718832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,127,0.10063999891281128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,127,0.10446400443712871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,255,0.21288534005482992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,127,0.1860533356666565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,255,0.13513066371281943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,255,0.11890666683514912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,255,0.15923200050989786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,255,0.12417067090670268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,255,0.11562666296958923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,255,0.14300266901652017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,255,0.15826132893562317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,255,0.21278399229049683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,255,0.12185600399971008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,255,0.13544000188509622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,255,0.12432533502578735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,255,0.11553600430488586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,255,0.11354666948318481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,511,0.20957332849502563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,511,0.18500266472498575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,511,0.167087992032369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,511,0.17553067207336426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,511,0.16404799620310465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,511,0.160970667997996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,511,0.3466133276621501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,511,0.1693120002746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,511,0.18468799193700156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,511,0.17364267508188883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,511,0.2089866598447164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,511,0.16437333822250366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,511,0.26690133412679035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,511,0.16200533509254456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,1023,0.2600319981575012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,1023,0.26713599761327106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,1023,0.27994134028752643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,1023,0.30742400884628296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,1023,0.25732266902923584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,1023,0.25410666068394977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,1023,0.5118879874547323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,1023,0.3847573200861613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,1023,0.26739199956258136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,1023,0.2626720070838928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,1023,0.30646934111913043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,1023,0.28010133902231854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,1023,0.254586656888326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,1023,0.2573653260866801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,2047,0.45074133078257245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,2047,0.4687733252843221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,2047,0.49418667952219647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,2047,0.4583306709925334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,2047,0.4431626796722412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,2047,0.44599465529123944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,2047,0.8606560230255127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,2047,0.45163198312123615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,2047,0.4691093365351359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,2047,0.49262932936350506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,2047,0.45553600788116455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,2047,0.44597331682840985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,2047,0.6089119911193848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,2047,0.4439093271891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,4095,1.510767936706543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,4095,0.8532000382741293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,4095,0.8746079603830973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,4095,0.9156213601430258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,4095,0.8285760084788004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,4095,0.8438506921132406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,4095,0.8236693541208903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,4095,0.8525706926981608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,4095,0.8751839796702067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,4095,1.0501973628997803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,4095,0.8406399885813395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,4095,0.8281919956207275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,4095,1.4377066294352214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,4095,0.8308959801991781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,8191,1.9186132748921711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,8191,1.5931199391682942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,8191,1.5758934020996094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,8191,1.6839893658955891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,8191,1.634650707244873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,8191,1.603424072265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,8191,2.5545652707417807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,8191,1.9144479433695476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,8191,1.6880639394124348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,8191,1.5738293329874675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,8191,1.6354613304138184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,8191,1.5724107424418132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,8191,1.6045440038045247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,8191,1.5890132586161296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,16383,5.3098398844401045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,16383,3.217402776082357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,16383,3.130703926086426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,16383,3.086890538533529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,16383,3.1084372202555337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,16383,3.066960016886393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,16383,3.0675201416015625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,16383,3.634496053059896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,16383,5.190405209859212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,16383,3.1699787775675454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,16383,3.0743627548217773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,16383,3.0634241104125977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,16383,3.1124852498372397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,16383,3.083477338155111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,1,32767,7.0715891520182295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,2,32767,6.1854508717854815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,8,32767,6.15878423055013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,16,32767,6.064736048380534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,32,32767,6.054543813069661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,64,32767,6.051514943440755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,1,32767,7.072965621948242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,512,1,4,32767,10.073514938354492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,1,0.21896000703175864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,1,0.2977386713027954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,1,0.1686613361040751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,1,0.1500640014807383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,1,0.13894933462142944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,1,0.13618666927019754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,1,0.13475733002026877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,2,32767,6.187386830647786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,1,0.2180160085360209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,1,0.2975359956423442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,1,0.14993600050608316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,1,0.13868799805641174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,1,0.16870933771133423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,4,32767,6.126058578491211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,1,0.13642666737238565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,1,0.1344586710135142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,3,0.14642133315404257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,3,0.16330666343371072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,8,32767,6.1545976003011065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,3,0.34085333347320557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,16,32767,6.06985600789388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,3,0.19346133867899576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,3,0.13537599643071493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,3,0.13757333159446716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,3,0.13340266545613608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,3,0.13727466265360513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,3,0.14893333117167154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,3,0.13467199603716531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,3,0.16129600008328757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,3,0.1934666633605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,32,32767,6.051024119059245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,3,0.2937386631965637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,512,1,64,32767,6.0514984130859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,3,0.1330560048421224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,7,0.1458186705907186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,7,0.16037333011627197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,7,0.13635200262069702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,7,0.13165866335233053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,7,0.2919573386510213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,7,0.1329866647720337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,7,0.22202134132385254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,7,0.16032532850901285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,7,0.1933013399442037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,7,0.144896000623703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,7,0.1357599993546804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,7,0.13405332962671915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,7,0.293722669283549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,7,0.13193066914876303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,15,0.15850133697191873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,15,0.13065600395202637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,15,0.13406399885813394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,15,0.19225066900253296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,15,0.1290666659673055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,15,0.1421173314253489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,15,0.13467199603716531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,15,0.14360533157984415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,15,0.16011733810106912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,15,0.13025066256523132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,15,0.19215466578801474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,15,0.34957865873972577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,15,0.12847466270128885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,15,0.29227733612060547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,31,0.13126400113105774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,31,0.14013333121935526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,31,0.16003732879956564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,31,0.1297920048236847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,31,0.12607466181119284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,31,0.1971679925918579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,31,0.2925119996070862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,31,0.1427839994430542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,31,0.127920001745224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,31,0.1569386621316274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,31,0.1270240048567454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,31,0.13453867038091025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,31,0.19608000914255777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,31,0.2932800054550171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,63,0.14242133498191833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,63,0.13266133268674216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,63,0.14012266198794046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,63,0.16190399726231894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,63,0.12599999705950418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,63,0.2093706727027893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,63,0.3023040095965068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,63,0.1602026621500651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,63,0.13235200444857279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,63,0.16168533762296042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,63,0.12784000237782797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,63,0.1288746694723765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,63,0.20643732945124307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,63,0.30101333061854046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,127,0.1519253353277842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,127,0.1519200007120768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,127,0.18870399395624796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,127,0.1465013325214386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,127,0.1648373305797577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,127,0.29263466596603394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,127,0.3436799844106038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,127,0.16305599610010782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,127,0.15452800194422403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,127,0.17318934202194214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,127,0.2336746652921041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,127,0.343669335047404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,127,0.14839466412862143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,127,0.18571732441584268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,255,0.19578667481740317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,255,0.20361600319544473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,255,0.23300800720850626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,255,0.28010133902231854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,255,0.19241599241892496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,255,0.27854400873184204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,255,0.3938133319218953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,255,0.2645973364512126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,255,0.2125493288040161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,255,0.19165333112080893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,255,0.23586666584014893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,255,0.27826666831970215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,255,0.19554666678110758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,255,0.3949226538340251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,511,0.3098133405049642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,511,0.5085013310114542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,511,0.28971733649571735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,511,0.28622400760650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,511,0.5327306588490804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,511,0.30107200145721436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,511,0.33285866181055707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,511,0.3104906678199768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,511,0.4119146664937337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,511,0.29662400484085083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,511,0.33340267340342206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,511,0.3783200184504191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,511,0.5077813466389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,511,0.2874880035718282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,1023,0.5000640153884888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,1023,0.4869226614634196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,1023,0.7859946886698405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,1023,0.4782986640930176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,1023,0.47389864921569824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,1023,0.5747893253962199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,1023,0.7273279825846354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,1023,0.4771360158920288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,1023,0.7526453336079916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,1023,0.4866186777750651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,1023,0.5248479843139648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,1023,0.5747466484705607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,1023,0.47443199157714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,1023,0.7277812957763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,2047,0.8733867009480795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,2047,0.8914453188578287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,2047,0.8536960283915201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,2047,0.8611253102620443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,2047,1.4432266553243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,2047,0.9938186804453532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,2047,1.17303466796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,2047,0.8752533594767252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,2047,0.892842690149943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,2047,0.928383986155192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,2047,1.1696053345998128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,2047,1.3543465932210286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,2047,0.8576906522115072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,2047,0.9837439854939779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,4095,2.0497546195983887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,4095,1.65447998046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,4095,1.6962827046712239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,4095,1.6306559244791667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,4095,2.5942986806233725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,4095,1.7721974054972331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,4095,1.6135466893513997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,4095,2.0470453898111978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,4095,1.6953120231628418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,4095,1.7699732780456543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,4095,1.6539039611816406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,4095,1.6029706001281738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,4095,1.6333707173665364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,4095,2.594266732533773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,8191,3.293738683064779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,8191,3.7779413859049478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,8191,3.2082080841064453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,8191,3.156789461771647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,8191,3.1214240392049155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,8191,3.1123520533243814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,8191,3.0983254114786782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,8191,5.524698893229167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,8191,3.300586700439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,8191,3.2365547815958657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,8191,3.157855987548828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,8191,3.1081387201944985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,8191,3.09663454691569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,8191,3.1253493626912436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,1,16383,7.21615473429362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,2,16383,10.29036776224772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,8,16383,6.215360005696614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,16,16383,6.11410649617513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,4,16383,6.2115523020426435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,32,16383,6.094207763671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,fp8,1024,1,64,16383,6.0814666748046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,1,16383,7.219664255777995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,2,16383,6.308693567911784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,4,16383,10.161616007486979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,16,16383,6.111258824666341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,8,16383,6.218965530395508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,32,16383,6.101184209187825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_generation,float16,float16,1024,1,64,16383,6.08247439066569
