framework,version,device,op_name,mla_dtype,kv_cache_dtype,batch_size,isl,tp_size,step,latency
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,1,0.022133332987626392
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,1,0.027141332626342773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,1,0.026517334083716076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,1,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,1,0.02645866572856903
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,1,0.027776000400384266
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,1,0.027002667387326557
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,1,0.02640533447265625
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,1,0.022181332111358643
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,1,0.025941332181294758
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,1,0.02604266752799352
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,1,0.02606933315594991
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,1,0.025914666553338368
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,1,0.02603733291228612
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,3,0.021744000415007275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,3,0.026426665484905243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,3,0.027957332630952198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,3,0.025920001169045765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,3,0.02622933437426885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,3,0.025744001070658367
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,3,0.02585600068171819
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,3,0.02163200080394745
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,3,0.02749866743882497
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,3,0.027477333943049114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,3,0.026714667677879333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,3,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,3,0.025839999318122864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,7,0.021525333325068157
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,3,0.025562666356563568
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,7,0.028042666614055634
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,7,0.02738133321205775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,7,0.026890667776266735
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,7,0.026122666895389557
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,7,0.026501332720120747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,7,0.027647999425729115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,7,0.026314665873845417
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,7,0.02223466585079829
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,7,0.027280000348885853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,7,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,7,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,7,0.026672000686327618
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,7,0.026560001075267792
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,15,0.022197333474953968
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,15,0.02757866680622101
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,15,0.026858667532602947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,15,0.026762666801611584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,15,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,15,0.02569599946339925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,15,0.021967999637126923
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,15,0.02626666675011317
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,15,0.027471999327341717
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,15,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,15,0.026965332527955372
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,15,0.02632533262173335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,15,0.026202666262785595
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,15,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,31,0.02161066730817159
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,31,0.027317332724730175
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,31,0.02603200078010559
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,31,0.026101333399613697
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,31,0.025744001070658367
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,31,0.025701334079106648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,31,0.025701334079106648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,31,0.02178666740655899
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,31,0.027232001225153606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,31,0.026501332720120747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,31,0.02569599946339925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,31,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,63,0.020389333367347717
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,31,0.025733334322770435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,31,0.025594666600227356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,63,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,63,0.025850666066010792
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,63,0.025040000677108765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,63,0.024879999458789825
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,63,0.024613333245118458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,63,0.020362666497627895
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,63,0.02463999887307485
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,63,0.025754667818546295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,63,0.024832000335057575
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,63,0.025418666501839954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,63,0.024538666009902954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,63,0.02463999887307485
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,63,0.024885334074497223
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,127,0.02867199977238973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,127,0.023210667073726654
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,127,0.02718399961789449
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,127,0.026858667532602947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,127,0.025936000049114227
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,127,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,127,0.02314666658639908
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,127,0.02869333326816559
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,127,0.027477333943049114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,127,0.02630399912595749
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,127,0.0268053337931633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,127,0.02603733291228612
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,127,0.02613866577545802
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,127,0.026159999271233875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,255,0.028560000161329906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,255,0.02734400083621343
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,255,0.023130667706330616
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,255,0.02661866694688797
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,255,0.026149332523345947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,255,0.02632533262173335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,255,0.023226665953795116
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,255,0.025957333544890087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,255,0.02871999889612198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,255,0.027727998793125153
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,255,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,255,0.02595199892918269
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,255,0.02643733223279317
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,255,0.0264533335963885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,511,0.023925334215164185
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,511,0.029285334050655365
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,511,0.028016000986099243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,511,0.026714667677879333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,511,0.027514666318893433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,511,0.0266239990790685
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,511,0.026757332185904186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,511,0.02385599911212921
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,511,0.029114666084448498
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,511,0.027994667490323383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,511,0.02734400083621343
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,511,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,511,0.026677332818508148
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,511,0.026517334083716076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,1023,0.025472000241279602
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,1023,0.028864001234372456
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,1023,0.028304000695546467
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,1023,0.03025600065787633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,1023,0.027978666126728058
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,1023,0.027845333019892376
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,1023,0.025445332129796345
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,1023,0.027664000789324444
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,1023,0.030213333666324615
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,1023,0.02882666637500127
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,1023,0.028202667832374573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,1023,0.02810666710138321
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,1023,0.02789866675933202
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,1023,0.027818667391935985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,2047,0.027776000400384266
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,2047,0.03324799984693527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,2047,0.03141866624355316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,2047,0.03051200012365977
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,2047,0.03089066594839096
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,2047,0.028090665737787884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,2047,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,2047,0.03027733415365219
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,2047,0.03310399999221166
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,2047,0.030954666435718536
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,2047,0.031530665854612984
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,2047,0.03048533449570338
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,2047,0.030453334252039593
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,2047,0.03030933439731598
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,4095,0.03763733307520548
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,4095,0.035786665976047516
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,4095,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,4095,0.03510933369398117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,4095,0.03475199888149897
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,4095,0.034389334420363106
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,4095,0.034661332766215004
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,4095,0.037647999823093414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,4095,0.030005333324273426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,4095,0.03581333408753077
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,4095,0.03515200068553289
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,4095,0.03486400097608566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,4095,0.03455466777086258
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,4095,0.03442666679620743
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,8191,0.03332266708215078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,8191,0.03972800076007843
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,8191,0.03818133225043615
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,8191,0.03675200045108795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,8191,0.03718933214743932
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,8191,0.0369759996732076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,8191,0.0335359995563825
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,8191,0.03973866750796636
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,8191,0.03681600093841553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,8191,0.036730666955312095
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,8191,0.03711466739575068
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,8191,0.03798400113979975
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,8191,0.03703466554482778
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,8191,0.03696533292531967
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,16383,0.038618666430314384
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,16383,0.048911998669306435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,16383,0.04494399825731913
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,16383,0.04531733194986979
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,16383,0.0452106644709905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,16383,0.04685866832733154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,16383,0.045066664616266884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,16383,0.038373333712418876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,16383,0.04666133224964142
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,16383,0.04863466819127401
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,16383,0.0452106644709905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,16383,0.04533866544564565
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,16383,0.04502933224042257
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,16383,0.0450133333603541
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,32767,0.04800533254941305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,32767,0.0581279993057251
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,32767,0.049738665421803795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,32767,0.05160533388455709
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,32767,0.04890666902065277
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,32767,0.048714667558670044
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,32767,0.048842668533325195
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,32767,0.04863999783992767
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,32767,0.05756799876689911
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,32767,0.051872000098228455
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,32767,0.048986668388048805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,32767,0.04901866614818573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,32767,0.04861866434415182
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,32767,0.04860800007979075
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,65535,0.06341333190600078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,65535,0.06528000036875407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,65535,0.06864533325036366
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,65535,0.06474133332570393
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,65535,0.06165333092212677
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,65535,0.0641599992911021
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,65535,0.0639626681804657
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,65535,0.06527466575304668
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,65535,0.06396799782911937
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,65535,0.06273599962393443
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,65535,0.06798933446407318
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,65535,0.06525866687297821
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,65535,0.06353599826494853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,65535,0.06182399888833364
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,4,131071,0.08364267150561015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,2,131071,0.08610666791598003
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,1,131071,0.08996267120043437
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,16,131071,0.07962666451931
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,8,131071,0.08179200192292531
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,32,131071,0.08074133098125458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1,1,64,131071,0.08028266827265422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,1,131071,0.09085333347320557
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,2,131071,0.08693333466847737
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,4,131071,0.08261333405971527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,8,131071,0.082096000512441
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,16,131071,0.07970666885375977
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,32,131071,0.08028799792130788
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1,1,64,131071,0.08051733175913493
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,1,0.027818667391935985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,1,0.022853332261244457
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,1,0.028309332827727
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,1,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,1,0.026922665536403656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,1,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,1,0.02293333411216736
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,1,0.026789332429567974
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,1,0.028357334434986115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,1,0.026954665780067444
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,1,0.027509334186712902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,1,0.02677333354949951
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,1,0.026901334524154663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,3,0.022624000906944275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,1,0.026341333985328674
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,3,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,3,0.027477333943049114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,3,0.026943999032179516
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,3,0.026714667677879333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,3,0.026613332331180573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,3,0.02678400029738744
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,3,0.022650666534900665
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,3,0.027989332874615986
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,3,0.027530667682488758
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,3,0.02697066714366277
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,3,0.026554666459560394
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,3,0.026634665826956432
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,3,0.02649066597223282
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,7,0.022634667654832203
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,7,0.02812266598145167
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,7,0.026933332284291584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,7,0.02682666728893916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,7,0.027114666998386383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,7,0.02632533262173335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,7,0.02664000044266383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,7,0.022656001150608063
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,7,0.026698666314284008
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,7,0.028202667832374573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,7,0.02754133443037669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,7,0.026778665681680042
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,7,0.02622933437426885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,7,0.026608000199000042
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,15,0.02222399910291036
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,15,0.026741333305835724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,15,0.027813332776228588
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,15,0.026757332185904186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,15,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,15,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,15,0.02585600068171819
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,15,0.022197333474953968
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,15,0.027957332630952198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,15,0.02679466704527537
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,15,0.026602665583292644
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,15,0.026485333840052288
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,15,0.0264533335963885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,15,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,31,0.02201066662867864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,31,0.027424000203609467
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,31,0.02664000044266383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,31,0.025920001169045765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,31,0.02587199956178665
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,31,0.021690666675567627
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,31,0.025936000049114227
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,31,0.025983999172846477
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,31,0.02703999976317088
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,31,0.026208000878492992
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,31,0.026672000686327618
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,31,0.02595199892918269
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,31,0.025360000630219776
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,31,0.02593066543340683
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,63,0.026362667481104534
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,63,0.020576000213623047
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,63,0.02526933451493581
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,63,0.025114665428797405
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,63,0.024677333732446034
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,63,0.026416001220544178
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,63,0.02422400067249934
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,63,0.025792000194390614
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,63,0.02054399996995926
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,63,0.02491733431816101
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,63,0.024874667326609295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,63,0.024890666206677754
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,63,0.02493866781393687
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,63,0.024826665719350178
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,127,0.027621333797772724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,127,0.023408000667889912
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,127,0.028922667105992634
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,127,0.0271519993742307
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,127,0.026858667532602947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,127,0.026565333207448322
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,127,0.02643733223279317
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,127,0.02347733328739802
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,127,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,127,0.028959999481836956
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,127,0.027237333357334137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,127,0.026949333647886913
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,127,0.026522666215896606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,127,0.02640533447265625
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,255,0.023733332753181458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,255,0.02924799919128418
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,255,0.027690666417280834
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,255,0.02717333287000656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,255,0.026901334524154663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,255,0.026565333207448322
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,255,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,255,0.023621333142121632
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,255,0.02905600021282832
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,255,0.02698666602373123
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,255,0.027722666660944622
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,255,0.02664000044266383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,255,0.026863999664783478
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,511,0.025237334271272022
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,255,0.02651199946800868
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,511,0.029872000217437744
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,511,0.02863466739654541
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,511,0.027930667002995808
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,511,0.027535999814669292
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,511,0.027786667148272198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,511,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,511,0.025274666647116344
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,511,0.030005333324273426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,511,0.028346667687098186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,511,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,511,0.02740799884001414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,511,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,511,0.027669332921504974
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,1023,0.02740799884001414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,1023,0.032186667124430336
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,1023,0.030789333085219067
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,1023,0.029919999341169994
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,1023,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,1023,0.029333333174387615
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,1023,0.029258665939172108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,1023,0.0308746670683225
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,1023,0.027258666853109997
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,1023,0.02991466720898946
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,1023,0.03249600032965342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,1023,0.02958933264017105
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,1023,0.029445332785447437
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,1023,0.02920000006755193
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,2047,0.029194665451844532
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,2047,0.03552533437808355
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,2047,0.03283733377854029
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,2047,0.032586666444937386
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,2047,0.0340639998515447
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,2047,0.032613334556420646
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,2047,0.032511999209721885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,2047,0.02898666759332021
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,2047,0.03568533311287562
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,2047,0.034416000048319496
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,2047,0.03303466737270355
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,2047,0.032442666590213776
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,4095,0.037290667494138084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,2047,0.032602667808532715
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,2047,0.03272533416748047
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,4095,0.03194133440653483
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,4095,0.03633599976698557
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,4095,0.035429333647092186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,4095,0.03495466709136963
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,4095,0.03490666548411051
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,4095,0.034661332766215004
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,4095,0.03777066618204117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,4095,0.03206400076548258
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,4095,0.03640000025431315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,4095,0.03514133393764496
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,4095,0.035317334036032356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,4095,0.03488533447186152
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,8191,0.036917333801587425
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,4095,0.034527999659379326
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,8191,0.040607998768488564
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,8191,0.04266133407751719
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,8191,0.039706667264302574
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,8191,0.03958400090535482
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,8191,0.03934400031963984
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,8191,0.03704000016053518
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,8191,0.039066667358080544
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,8191,0.04186666508515676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,8191,0.039861333866914116
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,8191,0.040565334260463715
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,8191,0.039605334401130676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,8191,0.03938133269548416
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,8191,0.03916800022125244
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,16383,0.04698666433493296
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,16383,0.04626133541266123
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,16383,0.05412800113360087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,16383,0.047279998660087585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,16383,0.04535466432571411
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,16383,0.04507199923197428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,16383,0.04484266539414724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,16383,0.04684799909591675
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,16383,0.05356800059477488
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,16383,0.048197334011395775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,16383,0.045365333557128906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,16383,0.04479999840259552
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,16383,0.046997333566347756
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,16383,0.04552533229192098
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,32767,0.061333333452542625
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,32767,0.06519466638565063
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,32767,0.06098666787147522
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,32767,0.060864001512527466
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,32767,0.0581279993057251
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,32767,0.06037333110968272
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,32767,0.059818665186564125
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,32767,0.060975998640060425
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,32767,0.06505600114663442
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,32767,0.062122667829195656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,32767,0.059658666451772056
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,32767,0.05807999769846598
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,32767,0.05959466596444448
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,32767,0.05959466596444448
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,65535,0.08904000123341878
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,65535,0.08370133241017659
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,65535,0.07991999884446462
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,65535,0.0788373351097107
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,65535,0.07634666562080383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,65535,0.07565866907437642
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,65535,0.07614933451016744
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,65535,0.0884320040543874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,65535,0.07950399816036224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,65535,0.0788213312625885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,65535,0.08365333080291748
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,65535,0.07745600243409474
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,65535,0.07573333382606506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,65535,0.07520000139872234
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,1,131071,0.14317867159843445
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,8,131071,0.11283733447392781
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,2,131071,0.11844799915949504
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,4,131071,0.11349333326021831
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,16,131071,0.11147733529408772
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,32,131071,0.11024533708890279
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,2,1,64,131071,0.10943466424942017
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,1,131071,0.1434399982293447
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,2,131071,0.11727466185887654
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,4,131071,0.1145919958750407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,8,131071,0.1125333309173584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,32,131071,0.11079999804496765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,16,131071,0.11204800009727478
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,2,1,64,131071,0.11046399672826131
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,1,0.023584000766277313
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,1,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,1,0.028357334434986115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,1,0.02759466568628947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,1,0.02719466636578242
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,1,0.02733866622050603
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,1,0.023333333432674408
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,1,0.028373333315054577
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,1,0.02717866748571396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,1,0.027045334378878277
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,1,0.02805333336194356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,1,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,1,0.0271519993742307
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,3,0.02829333394765854
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,3,0.02316266546646754
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,1,0.02714666724205017
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,3,0.027914665639400482
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,3,0.027136000494162243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,3,0.02701866626739502
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,3,0.027221334477265675
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,3,0.02332266668478648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,3,0.02701333413521449
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,3,0.028416000306606293
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,3,0.027957332630952198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,3,0.02738133321205775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,3,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,3,0.026917333404223125
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,3,0.027109332382678986
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,7,0.0232640008131663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,7,0.02775999903678894
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,7,0.028512001037597656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,7,0.02738133321205775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,7,0.026704000929991405
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,7,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,7,0.026922665536403656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,7,0.027973333994547527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,7,0.02279466638962428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,7,0.027690666417280834
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,7,0.027141332626342773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,7,0.02683199942111969
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,7,0.026778665681680042
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,7,0.026522666215896606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,15,0.022800001005331676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,15,0.027818667391935985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,15,0.02743999908367793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,15,0.027002667387326557
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,15,0.026682667434215546
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,15,0.02643733223279317
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,15,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,15,0.0230880007147789
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,15,0.026522666215896606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,15,0.028186666468779247
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,15,0.02693866689999898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,15,0.02664533257484436
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,15,0.026554666459560394
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,15,0.026533332963784535
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,31,0.022511998812357586
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,31,0.0271573339899381
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,31,0.026496000587940216
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,31,0.02682666728893916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,31,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,31,0.026191999514897663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,31,0.02611733227968216
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,31,0.0220320001244545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,31,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,31,0.026506667335828144
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,31,0.026464000344276428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,31,0.02629333237806956
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,31,0.0262719988822937
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,63,0.02548266698916753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,31,0.02613866577545802
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,63,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,63,0.020938667158285778
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,63,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,63,0.024703999360402424
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,63,0.02516266703605652
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,63,0.025290665527184803
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,63,0.021269333859284718
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,63,0.025968000292778015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,63,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,63,0.02514133354028066
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,63,0.025439999997615814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,63,0.02477866659561793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,63,0.025242666403452556
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,127,0.02380799998839696
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,127,0.029130667448043823
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,127,0.02738133321205775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,127,0.028149334092934925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,127,0.027215999861558277
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,127,0.02700799951950709
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,127,0.026682667434215546
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,127,0.02940800040960312
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,127,0.02792000025510788
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,127,0.023999998966852825
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,127,0.027050666511058807
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,127,0.027424000203609467
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,127,0.026880001028378803
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,127,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,255,0.028410665690898895
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,255,0.027552001178264618
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,255,0.029674666623274486
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,255,0.025231999655564625
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,255,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,255,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,255,0.027210667729377747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,255,0.02515200028816859
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,255,0.02959999938805898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,255,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,255,0.027994667490323383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,255,0.027509334186712902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,255,0.027290667096773785
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,255,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,511,0.027130665878454845
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,511,0.032127998769283295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,511,0.03047466774781545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,511,0.029877332349618275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,511,0.029301332930723827
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,511,0.028922667105992634
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,511,0.028912000358104706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,511,0.027189334233601887
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,511,0.02935466667016347
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,511,0.029557332396507263
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,511,0.03054933249950409
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,511,0.03203733265399933
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,511,0.029109333952267964
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,511,0.028768000503381092
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,1023,0.029194665451844532
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,1023,0.035061334570248924
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,1023,0.03320533285538355
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,1023,0.03209600100914637
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,1023,0.031888000667095184
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,1023,0.0317493329445521
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,1023,0.031744000812371574
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,1023,0.035002666215101876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,1023,0.029077333708604176
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,1023,0.032501332461833954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,1023,0.03156800071398417
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,1023,0.03323733309904734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,1023,0.03163733333349228
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,1023,0.03175999969244003
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,2047,0.032069332897663116
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,2047,0.03711999952793121
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,2047,0.03426666557788849
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,2047,0.03502399971087774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,2047,0.03365866591533025
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,2047,0.03390933324893316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,2047,0.03384533276160558
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,2047,0.031856000423431396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,2047,0.03705599904060364
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,2047,0.03443733354409536
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,2047,0.03526933242877325
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,2047,0.03348266581694285
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,2047,0.03390933324893316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,2047,0.03385599950949351
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,4095,0.03604800005753835
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,4095,0.04101866732041041
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,4095,0.03972266614437103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,4095,0.0384853333234787
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,4095,0.03783999880154928
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,4095,0.038021333515644073
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,4095,0.037690666814645134
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,4095,0.036144000788529716
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,4095,0.04102399945259094
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,4095,0.03928533444801966
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,4095,0.03834133346875509
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,4095,0.03801066676775614
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,4095,0.037658666570981346
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,4095,0.03788800040880839
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,8191,0.047653332352638245
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,8191,0.050714666644732155
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,8191,0.0460746685663859
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,8191,0.04389866689840952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,8191,0.043365334471066795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,8191,0.044064000248909
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,8191,0.04248533149560293
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,8191,0.0473280002673467
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,8191,0.05017066498597463
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,8191,0.04576533536116282
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,8191,0.044768000642458596
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,8191,0.043322667479515076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,8191,0.043477331598599754
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,8191,0.043280000487963356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,16383,0.061306665341059365
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,16383,0.06343466540177663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,16383,0.059104000528653465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,16383,0.05914666752020518
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,16383,0.058090666929880776
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,16383,0.05890133480230967
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,16383,0.05773333211739858
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,16383,0.06274133423964183
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,16383,0.06076799829800924
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,16383,0.06075733403364817
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,16383,0.058890665570894875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,16383,0.057328000664711
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,16383,0.057461331288019814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,16383,0.058186665177345276
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,32767,0.088128000497818
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,32767,0.0811359981695811
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,32767,0.07820799946784973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,32767,0.07628266513347626
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,32767,0.07559466858704884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,32767,0.07373333474000295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,32767,0.08799466490745544
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,32767,0.07528533538182576
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,32767,0.08177066842714946
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,32767,0.07668800155321757
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,32767,0.07594133416811626
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,32767,0.07564799984296162
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,32767,0.07542933523654938
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,65535,0.1425920029481252
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,65535,0.1092639962832133
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,32767,0.07401599983374278
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,65535,0.11639466881752014
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,65535,0.11198400457700093
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,65535,0.10935466488202412
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,65535,0.10878933469454448
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,65535,0.10797866185506184
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,65535,0.11501333117485046
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,65535,0.1425386667251587
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,65535,0.11126933495203654
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,65535,0.11016533772150676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,65535,0.11018666625022888
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,65535,0.1088800032933553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,65535,0.1079200009504954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,2,131071,0.1835626761118571
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,1,131071,0.2494773268699646
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,4,131071,0.17934934298197427
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,8,131071,0.17643199364344278
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,16,131071,0.17677332957585654
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,32,131071,0.1750239928563436
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,4,1,64,131071,0.17482133706410727
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,1,131071,0.24928534030914307
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,2,131071,0.18292800585428873
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,4,131071,0.18056533734003702
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,8,131071,0.17651732762654623
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,16,131071,0.17554134130477905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,1,0.02399466683467229
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,64,131071,0.17550400892893472
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,4,1,32,131071,0.17501866817474365
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,1,0.028090665737787884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,1,0.029487999776999157
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,1,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,1,0.027647999425729115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,1,0.027349332968393963
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,1,0.027610667049884796
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,1,0.024170666933059692
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,1,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,1,0.028079998989899952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,1,0.028064000109831493
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,1,0.02757333219051361
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,1,0.0276053324341774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,3,0.02386666586001714
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,1,0.027280000348885853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,3,0.029258665939172108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,3,0.028373333315054577
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,3,0.02739733209212621
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,3,0.02737066646416982
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,3,0.02758399893840154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,3,0.023887999355793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,3,0.028255999088287354
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,3,0.02735999971628189
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,3,0.028912000358104706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,3,0.027984000742435455
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,3,0.02703999976317088
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,3,0.027461332579453785
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,7,0.023733332753181458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,3,0.027530667682488758
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,7,0.029050665597120922
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,7,0.028330666323502857
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,7,0.02735466758410136
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,7,0.02738133321205775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,7,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,7,0.02734400083621343
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,7,0.023749334116776783
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,7,0.029167999823888142
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,7,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,7,0.02734400083621343
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,7,0.027834666272004444
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,7,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,7,0.027061333258946735
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,15,0.02889599899450938
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,15,0.023578666150569916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,15,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,15,0.02758399893840154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,15,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,15,0.02387733260790507
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,15,0.026757332185904186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,15,0.027082666754722595
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,15,0.02871999889612198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,15,0.02777066578467687
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,15,0.027562665442625683
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,15,0.02717866748571396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,15,0.02665599932273229
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,15,0.027098665634791057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,31,0.023333333432674408
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,31,0.028421332438786823
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,31,0.027509334186712902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,31,0.02666666607062022
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,31,0.026613332331180573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,31,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,31,0.026975999275843304
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,31,0.02274666726589203
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,31,0.027461332579453785
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,31,0.027994667490323383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,31,0.026943999032179516
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,31,0.026133333643277485
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,31,0.026709333062171936
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,31,0.026613332331180573
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,63,0.02181333303451538
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,63,0.027280000348885853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,63,0.02628266563018163
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,63,0.025813333690166473
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,63,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,63,0.024864000578721363
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,63,0.025466665625572205
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,63,0.021856000026067097
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,63,0.02741333345572154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,63,0.026394667724768322
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,63,0.025973332424958546
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,63,0.025839999318122864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,127,0.025498665869235992
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,63,0.024890666206677754
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,63,0.02510933329661687
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,127,0.030154667794704437
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,127,0.02871999889612198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,127,0.02845866729815801
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,127,0.028010666370391846
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,127,0.02756800005833308
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,127,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,127,0.029994666576385498
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,127,0.025418666501839954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,127,0.028736000259717304
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,127,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,127,0.02807466685771942
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,127,0.02758399893840154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,255,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,127,0.027888000011444092
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,255,0.032101333141326904
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,255,0.030346666773160298
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,255,0.029669334491093952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,255,0.02938133229811986
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,255,0.028991999725500744
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,255,0.02888533224662145
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,255,0.02733866622050603
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,255,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,255,0.03216533362865448
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,255,0.02961066613594691
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,255,0.02922133356332779
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,255,0.029109333952267964
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,255,0.028853334486484528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,511,0.02942399928967158
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,511,0.03494933247566223
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,511,0.033039999504884086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,511,0.0322080006202062
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,511,0.03143999973932902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,511,0.03162666658560435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,511,0.03140799949566523
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,511,0.02938133229811986
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,511,0.03510399907827377
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,511,0.033039999504884086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,511,0.03212266663710276
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,511,0.03183466692765554
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,511,0.03108799954255422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,511,0.03145066648721695
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,1023,0.032085334261258446
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,1023,0.035317334036032356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,1023,0.037018666664759316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,1023,0.03417599946260452
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,1023,0.03363733241955439
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,1023,0.03365333378314972
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,1023,0.03333866596221924
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,1023,0.03229333211978277
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,1023,0.03723733375469843
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,1023,0.03512533257404963
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,1023,0.03421866645415624
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,1023,0.033887999753157295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,1023,0.03350933392842611
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,1023,0.03344533344109853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,2047,0.03719999889532725
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,2047,0.04074133435885111
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,2047,0.03904533386230469
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,2047,0.03810133288304011
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,2047,0.037861332297325134
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,2047,0.03738133360942205
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,2047,0.03712533414363861
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,2047,0.04030933231115341
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,2047,0.03777066618204117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,2047,0.037765334049860634
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,2047,0.03812800099452337
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,2047,0.03903999924659729
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,4095,0.04994666576385498
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,2047,0.03766400118668874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,2047,0.03724266588687897
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,4095,0.04711466530958811
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,4095,0.04600533346335093
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,4095,0.04320000112056732
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,4095,0.04244266450405121
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,4095,0.04261333247025808
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,4095,0.047242666284243263
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,4095,0.04195733368396759
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,4095,0.05021866659323374
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,4095,0.045034666856129967
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,4095,0.04354133208592733
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,4095,0.04279466470082601
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,4095,0.04273599882920583
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,4095,0.043050666650136314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,8191,0.0620000014702479
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,8191,0.05816533168156942
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,8191,0.058005332946777344
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,8191,0.062319998939832054
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,8191,0.05806933343410492
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,8191,0.056421334544817604
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,8191,0.056218668818473816
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,8191,0.06228266656398773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,8191,0.06182933350404104
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,8191,0.05808533231417338
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,8191,0.05745066702365875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,8191,0.05927466849486033
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,8191,0.05552533268928528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,8191,0.056703999638557434
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,16383,0.08852266271909077
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,16383,0.08108266691366832
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,16383,0.07658133407433827
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,16383,0.07489599784215291
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,16383,0.07523199915885925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,16383,0.0746559997399648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,16383,0.0732426643371582
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,16383,0.0883733332157135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,16383,0.08117333551247914
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,16383,0.07754133145014445
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,16383,0.07424533367156982
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,16383,0.07423999905586243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,16383,0.07419733206431071
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,16383,0.07433600227038066
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,32767,0.1156160036722819
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,32767,0.11145066221555074
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,32767,0.10913599530855815
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,32767,0.14332800110181174
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,32767,0.10751466949780782
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,32767,0.10826133688290913
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,32767,0.1076746682325999
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,32767,0.14307199915250143
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,32767,0.11583466331164043
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,32767,0.11185600360234578
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,32767,0.10920000076293945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,32767,0.10705600182215373
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,32767,0.10643733541170756
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,32767,0.10771200060844421
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,65535,0.2493706742922465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,65535,0.18493866920471191
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,65535,0.1806559960047404
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,65535,0.17761600017547607
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,65535,0.17826133966445923
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,65535,0.17548799514770508
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,65535,0.17505600055058798
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,65535,0.24952000379562378
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,65535,0.18504534165064493
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,65535,0.18039999405543009
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,65535,0.17907732725143433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,65535,0.17782400051752725
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,65535,0.17465599377950033
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,65535,0.1758506695429484
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,1,131071,0.464901328086853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,2,131071,0.31645333766937256
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,4,131071,0.3118666609128316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,8,131071,0.3093493382136027
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,32,131071,0.3078719973564148
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,16,131071,0.3083786765734355
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,8,1,64,131071,0.30612800518671673
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,1,131071,0.46327467759450275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,2,131071,0.3166559934616089
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,4,131071,0.3109653393427531
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,16,131071,0.30763200918833417
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,8,131071,0.30914666255315143
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,32,131071,0.3080586592356364
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,8,1,64,131071,0.30719999472300213
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,1,0.026837334036827087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,1,0.02923733244339625
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,1,0.030586667358875275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,1,0.028938665986061096
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,1,0.02881066749493281
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,1,0.02870933214823405
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,1,0.026447998980681103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,1,0.02849599967400233
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,1,0.02920000006755193
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,1,0.030975999931494396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,1,0.02906133234500885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,1,0.028789333999156952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,1,0.028629332780838013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,1,0.028586665789286297
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,3,0.0262719988822937
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,3,0.029135999580224354
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,3,0.030533333619435627
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,3,0.02855466554562251
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,3,0.028618666032950085
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,3,0.02855466554562251
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,3,0.028522667785485584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,3,0.0262773334980011
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,3,0.028933333853880566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,3,0.031285333136717476
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,3,0.028965334097544353
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,3,0.028309332827727
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,3,0.02849599967400233
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,3,0.02847466617822647
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,7,0.030426666140556335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,7,0.025909334421157837
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,7,0.028864001234372456
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,7,0.02867199977238973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,7,0.028549333413441975
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,7,0.027973333994547527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,7,0.028378665447235107
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,7,0.026186667382717133
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,7,0.030415999392668407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,7,0.02827200045188268
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,7,0.029253333806991577
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,7,0.02826666583617528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,7,0.028037334481875103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,7,0.02829333394765854
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,15,0.03017599880695343
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,15,0.025701334079106648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,15,0.028101332485675812
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,15,0.02897600084543228
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,15,0.02805333336194356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,15,0.02804800122976303
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,15,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,15,0.02757333219051361
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,15,0.028037334481875103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,15,0.03012266755104065
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,15,0.02899733434120814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,15,0.028069332242012024
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,15,0.028005334238211315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,15,0.02809600035349528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,31,0.025429333249727886
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,31,0.028090665737787884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,31,0.029045333464940388
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,31,0.02754133443037669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,31,0.027855999767780304
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,31,0.028010666370391846
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,31,0.027632000545660656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,31,0.025221332907676697
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,31,0.029648000995318096
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,31,0.02805333336194356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,31,0.027402666707833607
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,31,0.027658666173617046
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,31,0.027482666075229645
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,63,0.024682665864626568
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,31,0.02741333345572154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,63,0.02874133239189784
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,63,0.026528000831604004
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,63,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,63,0.026330667237440746
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,63,0.02480533222357432
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,63,0.02657066782315572
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,63,0.02672533442576726
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,63,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,63,0.02754133443037669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,63,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,63,0.02657066782315572
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,63,0.026021334032217663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,63,0.026447998980681103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,127,0.032474666833877563
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,127,0.026554666459560394
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,127,0.030826665461063385
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,127,0.030224000414212544
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,127,0.029951999584833782
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,127,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,127,0.029690665503342945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,127,0.026447998980681103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,127,0.0310506671667099
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,127,0.032602667808532715
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,127,0.02998399982849757
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,127,0.02972800036271413
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,127,0.03014400104681651
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,255,0.03266133368015289
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,127,0.029834667841593426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,255,0.03488533447186152
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,255,0.03294399877389272
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,255,0.03235200047492981
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,255,0.03179733455181122
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,255,0.03205333401759466
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,255,0.031658666829268135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,255,0.03263466556866964
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,255,0.0345920001467069
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,255,0.033071999748547874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,255,0.032170665760835014
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,255,0.032127998769283295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,255,0.03161599983771642
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,255,0.03147733211517334
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,511,0.034901333351929985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,511,0.03746666759252548
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,511,0.03549866626660029
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,511,0.03500800083080927
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,511,0.034272000193595886
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,511,0.03342933456103007
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,511,0.03364799916744232
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,511,0.034714666505654655
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,511,0.03828266759713491
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,511,0.0346666673819224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,511,0.03526400029659271
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,511,0.03436266630887985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,511,0.033439998825391136
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,511,0.03402133285999298
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,1023,0.04275199770927429
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,1023,0.04041066765785217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,1023,0.04058133314053217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,1023,0.03913066784540812
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,1023,0.03852266569932302
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,1023,0.03812266637881597
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,1023,0.03770133356253306
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,1023,0.04260799785455068
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,1023,0.04028266668319702
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,1023,0.040207999447981514
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,1023,0.03909866760174433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,1023,0.03892799963553747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,1023,0.037920000652472176
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,1023,0.03833066672086716
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,2047,0.05004266897837321
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,2047,0.045824001232783
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,2047,0.053039997816085815
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,2047,0.04398933549722036
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,2047,0.043882668018341064
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,2047,0.04334933559099833
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,2047,0.04270400106906891
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,2047,0.04971733192602793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,2047,0.05121066669623057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,2047,0.04612799982229868
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,2047,0.04414933423201243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,2047,0.044682666659355164
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,2047,0.0435146689414978
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,2047,0.043765331308046974
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,4095,0.06498666604359944
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,4095,0.0631520003080368
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,4095,0.058703998724619545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,4095,0.05832533538341522
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,4095,0.058559998869895935
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,4095,0.057248001297314964
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,4095,0.05667733152707418
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,4095,0.0628959983587265
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,4095,0.0591893345117569
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,4095,0.06446933249632518
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,4095,0.05729066828886668
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,4095,0.05726400017738342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,4095,0.05820799867312113
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,4095,0.05706666906674703
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,8191,0.09179733196894328
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,8191,0.08296533425649007
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,8191,0.07825600107510884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,8191,0.07435200115044911
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,8191,0.0765119989713033
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,8191,0.07436800003051758
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,8191,0.07427733143170674
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,8191,0.08264000217119853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,8191,0.09114666779836018
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,8191,0.07621333499749501
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,8191,0.07752533257007599
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,8191,0.07412266731262207
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,8191,0.07495466868082683
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,8191,0.07411733269691467
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,16383,0.11570666233698527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,16383,0.14546666542689005
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,16383,0.11006399989128113
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,16383,0.10818666219711304
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,16383,0.11246400078137715
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,16383,0.10673066973686218
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,16383,0.10842667023340861
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,16383,0.14552533626556396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,16383,0.11628799637158711
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,16383,0.10718933741251628
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,16383,0.10930666327476501
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,16383,0.11166933178901672
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,16383,0.1071626643339793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,16383,0.10898133118947347
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,32767,0.1857759952545166
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,32767,0.2543039917945862
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,32767,0.18145600954691568
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,32767,0.1774453322092692
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,32767,0.17756267388661703
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,32767,0.17493865887324014
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,32767,0.17534399032592773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,32767,0.2537386616071065
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,32767,0.18700265884399414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,32767,0.18143999576568604
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,32767,0.1755146582921346
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,32767,0.17919999361038208
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,32767,0.17668799559275308
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,32767,0.1742666761080424
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,65535,0.3205066720644633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,65535,0.46747199694315594
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,65535,0.3130613366762797
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,65535,0.3096853295962016
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,65535,0.307370662689209
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,65535,0.30844799677530926
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,65535,0.30727465947469074
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,65535,0.46803200244903564
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,65535,0.32018133004506427
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,65535,0.3147413333257039
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,65535,0.31041600306828815
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,65535,0.3097013235092163
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,65535,0.30714666843414307
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,65535,0.3075786630312602
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,1,131071,0.8960800170898438
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,2,131071,0.585482676823934
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,4,131071,0.5791146755218506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,8,131071,0.5750720103581747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,64,131071,0.5717600186665853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,16,131071,0.5742506583531698
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,16,1,32,131071,0.5740746657053629
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,1,131071,0.8952960173288981
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,2,131071,0.5874559879302979
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,4,131071,0.5781600077946981
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,1,0.03419200082619985
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,1,0.02961066613594691
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,16,131071,0.5756640036900839
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,32,131071,0.5723839998245239
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,8,131071,0.5765706698099772
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,16,1,64,131071,0.5722773472468058
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,1,0.030762667457262676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,1,0.03224000086386999
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,1,0.031045332551002502
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,1,0.030229332546393078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,1,0.030821333328882854
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,1,0.030879999200503033
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,1,0.0340693344672521
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,1,0.03215466688076655
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,1,0.030981334547201794
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,1,0.030778666337331135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,1,0.031002665559450786
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,1,0.03053866575161616
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,3,0.029648000995318096
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,3,0.03396799912055334
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,3,0.0322826678554217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,3,0.03141866624355316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,3,0.030400000512599945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,3,0.031040000418821972
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,3,0.030954666435718536
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,3,0.03001066545645396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,3,0.0340693344672521
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,3,0.03226666649182638
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,3,0.031498665610949196
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,3,0.03054933249950409
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,3,0.03084266682465871
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,3,0.03075733284155528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,7,0.029440000653266907
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,7,0.033370666205883026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,7,0.031125334401925404
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,7,0.03186666717131933
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,7,0.030853333572546642
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,7,0.030261332790056866
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,7,0.030640001098314922
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,7,0.029498666524887085
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,7,0.031514666974544525
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,7,0.0336053321758906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,7,0.031290667752424874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,7,0.03035199890534083
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,7,0.03070933371782303
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,7,0.030773334205150604
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,15,0.03126933425664902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,15,0.033520000676314034
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,15,0.028944000601768494
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,15,0.030693332354227703
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,15,0.030368000268936157
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,15,0.030005333324273426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,15,0.03001066545645396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,15,0.033157333731651306
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,15,0.03165333221356074
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,15,0.029189333319664
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,15,0.03088533381621043
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,15,0.030506665507952373
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,15,0.030320001145203907
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,15,0.02993600070476532
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,31,0.02846933404604594
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,31,0.03268799930810928
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,31,0.031152000029881794
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,31,0.030213333666324615
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,31,0.02996266633272171
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,31,0.02995733420054118
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,31,0.029722665747006733
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,31,0.03311466674009959
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,31,0.02903466671705246
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,31,0.030373332401116688
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,31,0.029818666477998097
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,31,0.030346666773160298
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,31,0.02993600070476532
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,31,0.029930666089057922
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,63,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,63,0.027903998891512554
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,63,0.029711998999118805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,63,0.03236266722281774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,63,0.028560000161329906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,63,0.028880000114440918
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,63,0.028794666131337483
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,63,0.02807466685771942
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,63,0.031925333042939506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,63,0.02880000074704488
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,63,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,63,0.02870933214823405
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,63,0.029002666473388672
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,63,0.02867199977238973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,127,0.03462400039037069
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,127,0.029877332349618275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,127,0.03213333338499069
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,127,0.031157332162062328
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,127,0.03193599979082743
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,127,0.031311998764673867
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,127,0.02978666623433431
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,127,0.03457066665093104
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,127,0.03150933235883713
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,127,0.03250666707754135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,127,0.03141866624355316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,127,0.03108799954255422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,127,0.03149333347876867
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,127,0.03142400085926056
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,255,0.04136000076929728
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,255,0.033301333586374916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,255,0.03801066676775614
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,255,0.03718933214743932
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,255,0.03659199923276901
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,255,0.036501333117485046
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,255,0.0366293340921402
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,255,0.03299733251333237
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,255,0.04128533353408178
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,255,0.038015998899936676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,255,0.03737599899371465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,255,0.03734400123357773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,255,0.03648533423741659
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,255,0.03658133248488108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,511,0.04442666471004486
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,511,0.046906664967536926
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,511,0.04044266790151596
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,511,0.03932266682386398
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,511,0.03910933434963226
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,511,0.03916800022125244
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,511,0.03807466725508372
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,511,0.046816001335779824
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,511,0.04401599864164988
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,511,0.03977066775163015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,511,0.03919466584920883
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,511,0.03905066599448522
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,511,0.0391146664818128
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,511,0.038362666964530945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,1023,0.05818133552869161
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,1023,0.04690133531888326
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,1023,0.054154664278030396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,1023,0.04487466812133789
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,1023,0.045194665590922035
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,1023,0.044821331898371376
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,1023,0.04518933097521464
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,1023,0.053360000252723694
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,1023,0.05715733269850413
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,1023,0.04655466477076212
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,1023,0.04576533536116282
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,1023,0.044810667634010315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,1023,0.045594667394955955
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,1023,0.04497600098450979
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,2047,0.06517866750558217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,2047,0.06122666597366333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,2047,0.06955199937025706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,2047,0.059861332178115845
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,2047,0.056874667604764305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,2047,0.058789332707722984
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,2047,0.058277333776156105
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,2047,0.06984533369541168
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,2047,0.06562666594982147
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,2047,0.05937600135803223
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,2047,0.06122666597366333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,2047,0.057722667853037514
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,4095,0.0972106655438741
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,2047,0.056799997886021934
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,2047,0.05801066756248474
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,4095,0.08495466907819112
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,4095,0.07637866834799449
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,4095,0.07836266855398814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,4095,0.07705600063006084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,4095,0.07525866727034251
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,4095,0.07467733323574066
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,4095,0.0844640036424001
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,4095,0.09796800216039021
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,4095,0.0790719985961914
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,4095,0.07503999769687653
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,4095,0.07620800038178761
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,4095,0.0755573312441508
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,4095,0.07473599910736084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,8191,0.15084800124168396
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,8191,0.11236266295115153
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,8191,0.11859200398127238
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,8191,0.10920533537864685
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,8191,0.11154133081436157
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,8191,0.10848533113797505
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,8191,0.10905067125956218
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,8191,0.1513973375161489
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,8191,0.11828266580899556
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,8191,0.11235733826955159
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,8191,0.11005333065986633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,8191,0.10884267091751099
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,8191,0.10966933767000835
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,16383,0.2603786587715149
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,8191,0.10899733503659566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,16383,0.18595200777053833
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,16383,0.1816640098889669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,16383,0.17748800913492838
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,16383,0.1755680044492086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,16383,0.17932265996932983
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,16383,0.2600853244463603
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,16383,0.1770026683807373
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,16383,0.1802133321762085
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,16383,0.18623467286427817
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,16383,0.17574399709701538
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,16383,0.1763733426729838
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,16383,0.1783199906349182
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,16383,0.17666133244832358
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,32767,0.4744480053583781
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,32767,0.3256426652272542
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,32767,0.31383999188741046
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,32767,0.30902934074401855
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,32767,0.31548800071080524
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,32767,0.3072959979375203
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,32767,0.30904533465703327
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,32767,0.4748160044352214
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,32767,0.3255893389383952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,32767,0.3170986572901408
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,32767,0.3098986744880676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,32767,0.3129066626230876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,32767,0.3086666663487752
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,32767,0.3067466616630554
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,65535,0.901360034942627
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,65535,0.5908799966176351
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,65535,0.583402673403422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,65535,0.5721333424250284
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,65535,0.5785280068715414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,65535,0.5723359982172648
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,65535,0.9031413396199545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,65535,0.5735253492991129
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,65535,0.5909386475880941
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,65535,0.5819679896036783
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,65535,0.5771626631418864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,65535,0.5725706815719604
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,65535,0.5736159880956014
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,65535,0.573199987411499
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,4,131071,1.1075680255889893
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,1,131071,1.7600266138712566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,2,131071,1.1197973092397053
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,8,131071,1.1035733222961426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,16,131071,1.1037226517995198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,64,131071,1.0964852968851726
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,32,1,32,131071,1.09880534807841
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,1,131071,1.7582027117411296
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,2,131071,1.1200693448384602
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,8,131071,1.103775978088379
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,16,131071,1.10480531056722
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,4,131071,1.105679988861084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,1,0.03709333389997482
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,1,0.039408000806967415
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,32,131071,1.0997386773427327
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,1,0.037045332292715706
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,32,1,64,131071,1.098842700322469
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,1,0.03562666724125544
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,1,0.035018667578697205
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,1,0.03440000116825104
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,1,0.0365226666132609
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,1,0.0347680002450943
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,1,0.03692800054947535
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,1,0.039647998909155525
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,1,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,1,0.034917332231998444
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,1,0.03489066660404205
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,1,0.034448000291983284
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,3,0.036650667587916054
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,3,0.0369759996732076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,3,0.038986665507157646
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,3,0.03579200059175491
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,3,0.03509333233038584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,3,0.03498666733503342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,3,0.034874667723973594
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,3,0.03692266593376795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,3,0.03704000016053518
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,3,0.039461334546407066
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,3,0.03532800078392029
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,3,0.03519999980926514
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,3,0.03501333296298981
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,3,0.03506666670242945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,7,0.038906666139761605
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,7,0.03634133438269297
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,7,0.03676799933115641
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,7,0.03551466763019562
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,7,0.034458667039871216
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,7,0.03472533325354258
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,7,0.03485333422819773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,7,0.0386559988061587
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,7,0.0363520011305809
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,7,0.03578133384386698
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,7,0.03677333394686381
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,7,0.03477333237727483
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,7,0.034517332911491394
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,7,0.03484266748030981
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,15,0.03860266755024592
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,15,0.03598399957021078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,15,0.0352906659245491
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,15,0.03622400015592575
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,15,0.03451200077931086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,15,0.03421333432197571
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,15,0.03401600072781245
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,15,0.03601066768169403
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,15,0.03522666543722153
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,15,0.036202666660149894
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,15,0.03844800094763438
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,15,0.03443199892838796
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,15,0.03429866582155228
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,15,0.03420799970626831
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,31,0.03606933355331421
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,31,0.03843733419974645
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,31,0.03347733368476232
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,31,0.03451200077931086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,31,0.035877334574858345
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,31,0.033930666744709015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,31,0.03368533402681351
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,31,0.035530666510264076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,31,0.035775999228159584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,31,0.033973333736260734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,31,0.03804266701141993
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,31,0.03395200024048487
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,31,0.03384533276160558
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,31,0.0337119996547699
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,63,0.03475199888149897
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,63,0.037791999677817024
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,63,0.03298133363326391
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,63,0.03504000107447306
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,63,0.03291733314593633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,63,0.03284800052642822
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,63,0.032501332461833954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,63,0.03482666611671448
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,63,0.03797333439191183
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,63,0.03500800083080927
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,63,0.032746667663256325
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,63,0.03289599965016047
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,63,0.03285333265860876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,63,0.03290133426586787
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,127,0.037317333122094475
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,127,0.04052799940109253
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,127,0.03741333385308584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,127,0.03509333233038584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,127,0.03639466563860575
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,127,0.035445332527160645
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,127,0.03601066768169403
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,127,0.03698666642109553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,127,0.037317333122094475
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,127,0.04001600046952566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,127,0.035029334326585136
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,127,0.036090667049090065
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,127,0.03573866685231527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,127,0.035445332527160645
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,255,0.04374399781227112
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,255,0.04062933226426443
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,255,0.04091199984153112
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,255,0.03965333352486292
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,255,0.0386613334218661
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,255,0.03955200066169103
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,255,0.03899200012286504
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,255,0.040378667414188385
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,255,0.04367466767628988
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,255,0.041093334555625916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,255,0.040005333721637726
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,255,0.03841600070397059
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,255,0.039349332451820374
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,255,0.03961600114901861
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,511,0.05273599922657013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,511,0.05101866523424784
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,511,0.049551998575528465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,511,0.04964800179004669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,511,0.061119998494784035
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,511,0.048469334840774536
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,511,0.051701332132021584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,511,0.04819199939568838
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,511,0.06163200239340464
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,511,0.054474666714668274
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,511,0.04858666658401489
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,511,0.04944533109664917
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,511,0.0484799991051356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,511,0.04842133323351542
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,1023,0.06591466565926869
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,1023,0.07470400134722392
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,1023,0.06714133421579997
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,1023,0.06282133360703786
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,1023,0.062447999914487205
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,1023,0.06251733501752217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,1023,0.06726400057474773
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,1023,0.061887999375661217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,1023,0.07528000076611836
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,1023,0.06284266710281372
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,1023,0.06680533289909363
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,1023,0.061903998255729675
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,1023,0.06302399933338165
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,1023,0.06246933341026306
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,2047,0.09443199634552002
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,2047,0.09291199843088786
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,2047,0.08503466844558716
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,2047,0.0816480020682017
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,2047,0.07939200103282928
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,2047,0.07921066880226135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,2047,0.07930666704972585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,2047,0.09232532978057861
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,2047,0.0926080048084259
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,2047,0.08000533282756805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,2047,0.08180800080299377
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,2047,0.0841439962387085
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,2047,0.07948266466458638
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,2047,0.07984533409277599
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,4095,0.14973333477973938
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,4095,0.12658133109410605
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,4095,0.11917333801587422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,4095,0.1157866617043813
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,4095,0.11413333813349406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,4095,0.11441066861152649
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,4095,0.11401066184043884
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,4095,0.11901332934697469
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,4095,0.14884266257286072
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,4095,0.1265653371810913
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,4095,0.1142080028851827
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,4095,0.11461866895357768
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,4095,0.11370133360226949
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,4095,0.1128053367137909
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,8191,0.19257599115371704
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,8191,0.25893332560857135
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,8191,0.18574400742848715
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,8191,0.1807466745376587
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,8191,0.18061333894729614
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,8191,0.1811413367589315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,8191,0.18045334021250406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,8191,0.25808533032735187
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,8191,0.19308799505233765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,8191,0.1814026633898417
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,8191,0.18116267522176108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,8191,0.18561599651972452
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,8191,0.18041600783665976
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,8191,0.17828265825907388
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,16383,0.4904053211212158
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,16383,0.3407520055770874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,16383,0.32602665821711224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,16383,0.31670933961868286
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,16383,0.3197439908981323
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,16383,0.31254400809605914
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,16383,0.31300799051920575
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,16383,0.49000000953674316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,16383,0.33990931510925293
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,16383,0.32769066095352173
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,16383,0.3175999919573466
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,16383,0.3166133364041646
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,16383,0.31463466087977093
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,16383,0.31328533093134564
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,32767,0.591327985127767
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,32767,0.9148106575012207
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,32767,0.6056319872538248
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,32767,0.5781866709391276
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,32767,0.577184001604716
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,32767,0.5832800070444742
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,32767,0.5775093237559
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,32767,0.91649063428243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,32767,0.6060160001118978
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,32767,0.5766613483428955
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,32767,0.5926666657129923
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,32767,0.5807573397954305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,32767,0.577674667040507
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,32767,0.577733318010966
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,65535,1.7718186378479004
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,65535,1.135818640391032
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,65535,1.119989315668742
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,65535,1.1050293445587158
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,65535,1.1115787029266357
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,65535,1.1027039686838787
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,65535,1.102453311284383
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,65535,1.770426591237386
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,65535,1.1366666952768962
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,65535,1.1199253400166829
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,65535,1.112165371576945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,65535,1.1063466866811116
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,65535,1.1037866274515789
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,65535,1.1029866536458333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,1,131071,3.4809707005818686
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,2,131071,2.186565399169922
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,32,131071,2.1533759435017905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,4,131071,2.17141326268514
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,8,131071,2.1597867012023926
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,16,131071,2.154080073038737
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,64,1,64,131071,2.151583989461263
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,1,131071,3.4798027674357095
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,2,131071,2.1889333724975586
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,1,0.053770666321118675
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,1,0.049546668926874794
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,4,131071,2.168010711669922
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,1,0.04569066564242045
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,8,131071,2.160271962483724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,1,0.0433599998553594
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,16,131071,2.1548585891723633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,32,131071,2.15230401357015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,64,1,64,131071,2.152165253957113
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,1,0.042266666889190674
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,1,0.0421013335386912
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,1,0.04155199974775314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,1,0.049423997600873314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,1,0.053616002202034
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,1,0.04534400006135305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,1,0.04237333436806997
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,1,0.04199466605981191
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,1,0.04338666796684265
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,1,0.04200000067551931
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,3,0.04944533109664917
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,3,0.05375466744105021
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,3,0.04516266783078512
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,3,0.042277331153551735
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,3,0.043391997615496315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,3,0.04242133100827535
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,3,0.041797334949175514
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,3,0.05299200117588043
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,3,0.04297066728274027
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,3,0.04951466619968414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,3,0.0452159990866979
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,3,0.042725334564844765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,3,0.04185600082079569
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,3,0.041984001795450844
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,7,0.049327999353408813
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,7,0.05296533306439718
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,7,0.04483733574549357
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,7,0.04314666489760081
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,7,0.04203199843565623
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,7,0.0418453315893809
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,7,0.04171733558177948
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,7,0.05298133194446564
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,7,0.04924266537030538
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,7,0.045237332582473755
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,7,0.04275199770927429
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,7,0.04221333563327789
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,7,0.041802664597829185
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,7,0.04164266586303711
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,15,0.052383999029795326
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,15,0.041450666884581246
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,15,0.04900266726811727
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,15,0.04489600161711375
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,15,0.04284800092379252
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,15,0.04119466741879781
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,15,0.04119999955097834
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,15,0.05348266661167145
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,15,0.04447466631730398
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,15,0.04177066683769226
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,15,0.048954665660858154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,15,0.042805333932240806
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,15,0.04134399940570196
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,15,0.04127466678619385
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,31,0.051957334081331887
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,31,0.047882666190465294
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,31,0.043840001026789345
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,31,0.04241600135962168
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,31,0.04130133241415024
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,31,0.040976000328858696
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,31,0.04081599911053976
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,31,0.04841599861780802
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,31,0.051642666260401406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,31,0.04372799893220266
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,31,0.041834667325019836
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,31,0.04139200101296107
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,31,0.040906667709350586
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,31,0.040709334115187325
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,63,0.0532533327738444
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,63,0.048245335618654885
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,63,0.043466667334238686
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,63,0.04016000032424927
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,63,0.04095999896526337
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,63,0.04008533308903376
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,63,0.04001600046952566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,63,0.052799999713897705
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,63,0.043663998444875084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,63,0.048021331429481506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,63,0.04156800111134847
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,63,0.03997866561015447
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,63,0.0395413339138031
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,63,0.03984533250331879
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,127,0.06005866825580597
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,127,0.04654400050640106
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,127,0.05115200082461039
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,127,0.04412800073623657
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,127,0.043381333351135254
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,127,0.04274666806062063
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,127,0.04268266757329305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,127,0.05979733169078827
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,127,0.04637333254019419
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,127,0.05072000126043955
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,127,0.04438399771849314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,127,0.04345599810282389
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,127,0.04278400043646494
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,127,0.042917331059773765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,255,0.060645331939061485
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,255,0.06843199829260509
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,255,0.05213333169619242
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,255,0.04644800225893656
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,255,0.04773333172003428
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,255,0.048485333720842995
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,255,0.046810666720072426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,255,0.067221333583196
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,255,0.06053866446018219
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,255,0.04882133503754934
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,255,0.046853333711624146
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,255,0.05213333169619242
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,255,0.047210668524106346
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,255,0.04692266881465912
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,511,0.08052266637484233
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,511,0.07075199981530507
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,511,0.06132266422112783
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,511,0.06532266736030579
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,511,0.0643039991458257
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,511,0.06123200058937073
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,511,0.061343997716903687
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,511,0.08086400230725606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,511,0.07083199918270111
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,511,0.06258133550484975
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,511,0.06197333335876465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,511,0.06612800061702728
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,511,0.062368000547091164
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,511,0.060266668597857155
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,1023,0.10765332976977031
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,1023,0.0886240005493164
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,1023,0.08312533299128215
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,1023,0.07986666758855183
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,1023,0.0796853353579839
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,1023,0.07903466622034709
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,1023,0.0790293316046397
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,1023,0.1088800032933553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,1023,0.08871466914812724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,1023,0.07852800190448761
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,1023,0.08360000451405843
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,1023,0.08036800225575765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,1023,0.07939733564853668
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,1023,0.0788373351097107
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,2047,0.16318399707476297
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,2047,0.12140267093976338
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,2047,0.11408533652623494
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,2047,0.11678933103879292
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,2047,0.11387200156847636
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,2047,0.11275200049082439
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,2047,0.11290666460990906
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,2047,0.1632426679134369
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,2047,0.12297067046165466
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,2047,0.11429867148399353
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,2047,0.11643200119336446
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,2047,0.11400533715883891
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,2047,0.11391466856002808
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,2047,0.11286399761835735
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,4095,0.27402667204538983
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,4095,0.19179733594258627
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,4095,0.18126400311787924
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,4095,0.1854026714960734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,4095,0.18172266085942587
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,4095,0.18130133549372354
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,4095,0.18094400564829508
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,4095,0.19051732619603476
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,4095,0.27484800418217975
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,4095,0.18503467241923013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,4095,0.1816693345705668
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,4095,0.18152532974878946
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,4095,0.1803200046221415
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,4095,0.18122132619222006
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,8191,0.3226240078608195
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,8191,0.5009386539459229
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,8191,0.3160960078239441
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,8191,0.31436800956726074
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,8191,0.31329600016276044
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,8191,0.3118506669998169
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,8191,0.31116799513498944
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,8191,0.4978080193201701
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,8191,0.32287466526031494
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,8191,0.3163786729176839
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,8191,0.31440534194310504
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,8191,0.31178667147954303
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,8191,0.31217066446940106
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,8191,0.3128746747970581
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,16383,0.6333386500676473
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,16383,0.9295039971669515
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,16383,0.6115573247273763
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,16383,0.5871146519978842
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,16383,0.5980586608250936
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,16383,0.5850880146026611
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,16383,0.5811946789423624
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,16383,0.9279359976450602
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,16383,0.6329333384831747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,16383,0.5980960130691528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,16383,0.6096160014470419
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,16383,0.5887999931971232
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,16383,0.5835306644439697
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,16383,0.5858026742935181
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,32767,1.7834186553955078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,32767,1.164736032485962
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,32767,1.1235146522521973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,32767,1.136847972869873
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,32767,1.1066826979319255
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,32767,1.106042702992757
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,32767,1.109434684117635
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,32767,1.7835680643717449
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,32767,1.1651413440704346
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,32767,1.136895974477132
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,32767,1.124303976694743
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,32767,1.113818645477295
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,32767,1.1098559697469075
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,32767,1.1058879693349202
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,65535,2.190645376841227
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,65535,2.2180533409118652
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,65535,3.495456059773763
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,65535,2.174832026163737
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,65535,2.165583928426107
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,65535,2.1576852798461914
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,65535,2.1610453923543296
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,65535,3.50058650970459
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,65535,2.2204480171203613
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,65535,2.1917333602905273
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,65535,2.1749119758605957
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,65535,2.158789316813151
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,65535,2.163834730784098
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,65535,2.1564586957295737
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,4,131071,4.284976005554199
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,32,131071,4.256981213887532
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,2,131071,4.318021456400554
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,1,131071,6.911626815795898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,1,131071,6.91326904296875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,64,131071,4.253493309020996
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,8,131071,4.269354820251465
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,128,1,16,131071,4.256976127624512
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,1,0.09124267101287842
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,2,131071,4.3134721120198565
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,1,0.0692799985408783
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,1,0.06172266602516174
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,1,0.05769599974155426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,4,131071,4.288618723551433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,1,0.05629333357016245
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,1,0.055904000997543335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,1,0.054976001381874084
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,1,0.09205333391825359
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,1,0.06964266796906789
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,1,0.0617439995209376
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,1,0.057674666245778404
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,1,0.05602133274078369
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,1,0.055813332398732506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,1,0.05554133156935374
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,3,0.09176533420880635
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,3,0.06922133266925812
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,3,0.05821333328882853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,3,0.061887999375661217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,64,131071,4.2496639887491865
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,8,131071,4.269232114156087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,32,131071,4.252592086791992
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,3,0.05532800157864889
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,128,1,16,131071,4.260501225789388
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,3,0.055488000313440956
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,3,0.05612266560395559
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,3,0.09096533060073853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,3,0.06168533364931742
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,3,0.058037335673967995
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,3,0.06952000161012013
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,3,0.056090667843818665
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,3,0.05524266759554545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,3,0.05588266750176748
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,7,0.06112533311049143
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,7,0.057861333092053734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,7,0.09041066964467366
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,7,0.05632533133029938
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,7,0.07057600220044453
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,7,0.05504000186920166
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,7,0.05513066550095876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,7,0.09029866258303325
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,7,0.06985066831111908
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,7,0.06133866806825002
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,7,0.0580213318268458
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,7,0.056234667698542275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,7,0.05559466779232025
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,7,0.05468800167242686
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,15,0.08970666925112407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,15,0.0701279987891515
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,15,0.057034666339556374
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,15,0.06035199761390686
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,15,0.05451733370621999
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,15,0.05522666871547699
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,15,0.054373333851496376
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,15,0.09060800075531006
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,15,0.06920533378918965
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,15,0.05704533557097117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,15,0.05495999753475189
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,15,0.06055466830730438
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,15,0.05453866720199585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,15,0.05401599903901418
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,31,0.08966933687527974
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,31,0.05366933345794678
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,31,0.05612266560395559
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,31,0.05973333120346069
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,31,0.053599998354911804
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,31,0.0690826674302419
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,31,0.053247998158137
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,31,0.08967467149098714
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,31,0.07022933165232341
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,31,0.05611733098824819
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,31,0.059018666545550026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,31,0.05455466608206431
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,31,0.053727999329566956
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,31,0.05275199810663859
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,63,0.09105066458384196
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,63,0.05455466608206431
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,63,0.07413866619269054
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,63,0.059450666109720864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,63,0.05218133330345154
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,63,0.051701332132021584
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,63,0.05206400156021118
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,63,0.09181333581606548
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,63,0.0739573339621226
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,63,0.058965335289637245
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,63,0.05416533350944519
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,63,0.05221333106358846
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,63,0.05276800195376078
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,63,0.05184000233809153
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,127,0.1014400025208791
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,127,0.0792799989382426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,127,0.06987200180689494
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,127,0.06325866778691609
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,127,0.0587360014518102
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,127,0.05692266424496969
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,127,0.05769599974155426
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,127,0.09843732913335164
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,127,0.07945066690444946
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,127,0.058677335580190025
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,127,0.06109866499900818
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,127,0.057717333237330117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,127,0.056890666484832764
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,127,0.06866133213043213
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,255,0.07839466631412506
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,255,0.11196800072987874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,255,0.08909866213798523
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,255,0.07538133362929027
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,255,0.0730453332265218
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,255,0.07233066856861115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,255,0.07130133112271626
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,255,0.08864532907803853
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,255,0.07543466488520305
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,255,0.07918400069077809
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,255,0.11337600151697795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,255,0.07249600191911061
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,255,0.07204799850781758
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,255,0.07150933146476746
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,511,0.1381226678689321
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,511,0.10781866312026978
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,511,0.09799999992052714
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,511,0.09330667058626811
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,511,0.08954133590062459
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,511,0.08962666988372803
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,511,0.13904000322024027
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,511,0.10900800426801045
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,511,0.09733866651852925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,511,0.08959466218948364
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,511,0.0936959981918335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,511,0.09089066584904988
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,511,0.08961600065231323
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,511,0.09046933054924011
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,1023,0.19493865966796875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,1023,0.14482133587201437
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,1023,0.1250933309396108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,1023,0.1237440009911855
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,1023,0.13154133160909018
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,1023,0.12359999616940816
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,1023,0.19353065888086954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,1023,0.12601066629091898
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,1023,0.14331199725468954
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,1023,0.1279306709766388
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,1023,0.1333013375600179
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,1023,0.12322133779525757
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,1023,0.12533866365750632
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,1023,0.12313066919644673
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,2047,0.3040320078531901
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,2047,0.1982240080833435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,2047,0.19027199347813925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,2047,0.19543466965357462
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,2047,0.19234667221705118
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,2047,0.1898933251698812
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,2047,0.3062880039215088
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,2047,0.20846933126449585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,2047,0.21001599232355753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,2047,0.20037867625554404
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,2047,0.19429334004720053
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,2047,0.19196265935897827
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,2047,0.1897546648979187
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,2047,0.19054933389027914
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,4095,0.5237760146458944
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,4095,0.3315466642379761
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,4095,0.32715733846028644
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,4095,0.3237920006116231
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,4095,0.3246133327484131
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,4095,0.3217759927113851
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,4095,0.34136001269022626
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,4095,0.5256533225377401
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,4095,0.34086398283640545
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,4095,0.3312373359998067
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,4095,0.3288693428039551
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,4095,0.3253386616706848
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,4095,0.3232480088869731
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,4095,0.3222879966100057
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,8191,0.6512053410212199
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,8191,0.6248693466186523
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,8191,0.6085813442866007
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,8191,0.590773344039917
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,8191,0.598853349685669
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,8191,0.5949440002441406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,8191,0.9512639840443929
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,8191,0.9510506788889567
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,8191,0.6529066562652588
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,8191,0.6087146600087484
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,8191,0.597765326499939
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,8191,0.5936959981918335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,8191,0.5902986526489258
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,8191,0.6228053172429403
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,16383,1.1894079844156902
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,16383,1.806821346282959
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,16383,1.1563680171966553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,16383,1.1145973205566406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,16383,1.1364479859670003
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,16383,1.129856030146281
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,16383,1.1187466780344646
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,16383,1.8060533205668132
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,16383,1.1904213428497314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,16383,1.1366346677144368
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,16383,1.155407985051473
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,16383,1.1175039609273274
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,16383,1.11899201075236
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,16383,1.1268426577250164
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,32767,3.5217971801757812
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,32767,2.247450669606527
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,32767,2.1910826365152993
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,32767,2.213306744893392
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,32767,2.170186678568522
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,32767,2.16321595509847
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,32767,2.1769866943359375
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,32767,3.525578816731771
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,32767,2.2475892702738443
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,32767,2.209669272104899
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,32767,2.1921653747558594
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,32767,2.1689279874165854
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,32767,2.1745173136393228
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,32767,2.164757410685221
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,1,65535,6.939381281534831
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,2,65535,4.351061185201009
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,8,65535,4.289893468221028
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,16,65535,4.275162696838379
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,4,65535,4.311637242635091
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,32,65535,4.264597256978353
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,256,1,64,65535,4.264117240905762
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,1,65535,6.9465281168619795
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,1,0.158160001039505
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,1,0.11389866471290588
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,1,0.09537600477536519
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,1,0.08748799562454224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,1,0.08433066805203755
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,1,0.08270933230717976
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,1,0.0820000022649765
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,1,0.1583093305428823
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,2,65535,4.350170771280925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,1,0.11405332883199056
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,1,0.09668266773223877
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,8,65535,4.287199974060059
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,16,65535,4.276613235473633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,1,0.0874079962571462
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,64,65535,4.2675520579020185
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,1,0.08427733182907104
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,1,0.0822026679913203
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,32,65535,4.263573328653972
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,1,0.08164266745249431
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,3,0.11337066690127055
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,3,0.09658132990201314
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,256,1,4,65535,4.313690821329753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,3,0.1570186714331309
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,3,0.08382933338483174
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,3,0.08715732892354329
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,3,0.08240533371766408
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,3,0.08189333478609721
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,3,0.08710400263468425
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,3,0.09569600224494934
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,3,0.08388266960779826
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,3,0.11364266276359558
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,3,0.08229866623878479
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,3,0.15683733423550925
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,3,0.08221866687138875
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,7,0.08354666829109192
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,7,0.08646399776140849
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,7,0.156031996011734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,7,0.09609599908192952
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,7,0.08146133522192638
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,7,0.1120799978574117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,7,0.08156799773375194
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,7,0.15590399503707886
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,7,0.08215466638406117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,7,0.08371733625729878
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,7,0.08679466446240743
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,7,0.08082133531570435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,7,0.09571199615796407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,7,0.11313600341478984
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,15,0.1551520029703776
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,15,0.11266133189201355
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,15,0.08177599807580312
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,15,0.07951466739177704
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,15,0.07964799801508586
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,15,0.08524266878763835
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,15,0.09497599800427754
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,15,0.1554026703039805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,15,0.11145066221555074
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,15,0.08161066472530365
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,15,0.08499733606974284
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,15,0.09494933485984802
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,15,0.07958399752775829
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,15,0.0795253316561381
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,31,0.15308266878128052
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,31,0.11357333262761433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,31,0.09268266956011455
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,31,0.07981333136558533
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,31,0.07830399771531422
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,31,0.08353066444396973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,31,0.07698133091131847
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,31,0.15291733543078104
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,31,0.1134986678759257
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,31,0.09469866752624512
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,31,0.08311999837557475
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,31,0.07836266855398814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,31,0.07966400186220805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,31,0.07766399780909221
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,63,0.15779200196266174
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,63,0.08876267075538635
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,63,0.09856533010800679
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,63,0.11674132943153381
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,63,0.07937066753705342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,63,0.07627733548482259
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,63,0.07406933108965556
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,63,0.15566933155059814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,63,0.0886293351650238
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,63,0.07542400062084198
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,63,0.11522666613260905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,63,0.07414400080839793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,63,0.0981173316637675
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,63,0.0806879997253418
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,127,0.1722453236579895
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,127,0.12678399682044983
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,127,0.09141866366068523
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,127,0.10859733819961548
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,127,0.09656533598899841
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,127,0.092031995455424
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,127,0.09128000338872273
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,127,0.17228800058364868
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,127,0.10885866483052571
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,127,0.1267680029074351
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,127,0.08990933497746785
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,127,0.09455999732017517
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,127,0.09101333220799764
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,127,0.09681066870689392
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,255,0.19779733816782633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,255,0.11000532905260722
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,255,0.10890666643778484
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,255,0.11559999982515971
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,255,0.12432000041007996
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,255,0.10864532987276714
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,255,0.14328533411026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,255,0.19991467396418253
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,255,0.14416533708572388
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,255,0.11382399996121724
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,255,0.12451733152071635
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,255,0.10957866907119751
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,255,0.11059733231862386
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,255,0.10913599530855815
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,511,0.252837340037028
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,511,0.1606880029042562
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,511,0.17941333850224814
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,511,0.14630400141080221
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,511,0.1439359982808431
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,511,0.1423733333746592
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,511,0.14965333541234335
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,511,0.25321600834528607
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,511,0.14643200238545737
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,511,0.1602026621500651
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,511,0.1502133309841156
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,511,0.1816320021947225
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,511,0.14412267009417215
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,511,0.14215466380119324
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,1023,0.36450668176015216
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,1023,0.24822932481765747
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,1023,0.21901865800221762
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,1023,0.2095680038134257
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,1023,0.22816532850265503
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,1023,0.21196800470352173
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,1023,0.2109066645304362
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,1023,0.36429333686828613
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,1023,0.25041600068410236
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,1023,0.22835199038187662
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,1023,0.21905599037806192
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,1023,0.211626668771108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,1023,0.2140106757481893
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,1023,0.20893333355585733
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,2047,0.5751413504282633
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,2047,0.3607413371404012
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,2047,0.38150401910146076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,2047,0.3455679814020793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,2047,0.3512800137201945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,2047,0.343231995900472
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,2047,0.34141866366068524
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,2047,0.5759093364079794
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,2047,0.380240003267924
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,2047,0.35979199409484863
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,2047,0.35042134920756024
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,2047,0.34774935245513916
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,2047,0.34275201956431073
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,2047,0.34061865011850995
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,4095,1.0012586911519368
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,4095,0.6813653310139974
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,4095,0.6290773153305054
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,4095,0.6106453339258829
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,4095,0.6503413518269857
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,4095,0.6060693264007568
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,4095,0.617845336596171
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,4095,1.0025973320007324
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,4095,0.6857333183288574
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,4095,0.6293813387552897
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,4095,0.6510026852289835
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,4095,0.6197439829508463
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,4095,0.6052906513214111
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,4095,0.6112693150838217
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,8191,1.2349653244018555
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,8191,1.8571306864420574
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,8191,1.1889866987864177
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,8191,1.139408032099406
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,8191,1.1602986653645833
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,8191,1.1442933082580566
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,8191,1.1321760018666585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,8191,1.854090690612793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,8191,1.2328533331553142
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,8191,1.1635359923044841
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,8191,1.1884480317433674
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,8191,1.1450453599294026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,8191,1.1384373505910237
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,8191,1.1327146689097087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,16383,3.5738347371419272
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,16383,2.2932799657185874
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,16383,2.2446559270222983
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,16383,2.2144692738850913
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,16383,2.188426653544108
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,16383,2.1972427368164062
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,16383,2.1820853551228843
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,16383,3.566447893778483
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,16383,2.2910133997599282
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,16383,2.2457173665364585
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,16383,2.218714714050293
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,16383,2.2017973264058432
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,16383,2.1909920374552407
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,16383,2.1815786361694336
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,1,32767,7.000821431477864
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,2,32767,4.405322710673015
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,16,32767,4.2966718673706055
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,64,32767,4.2818559010823565
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,32,32767,4.281680107116699
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,8,32767,4.315845489501953
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,512,1,4,32767,4.34873612721761
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,1,32767,6.985525131225586
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,1,0.1939093271891276
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,1,0.2937120000521342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,1,0.16369066635767618
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,1,0.1492693324883779
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,1,0.13692800203959146
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,1,0.1393226683139801
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,1,0.13540800412495932
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,2,32767,4.405237197875977
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,4,32767,4.348890622456868
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,1,0.2942986687024434
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,1,0.19380799929300943
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,1,0.1393173336982727
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,1,0.16247466206550598
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,1,0.14789866407712302
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,1,0.1365386644999186
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,1,0.13530666629473367
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,3,0.16263467073440552
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,3,0.19074666500091553
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,8,32767,4.316213289896647
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,3,0.14748799800872803
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,3,0.29042667150497437
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,3,0.1386293371518453
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,3,0.13497066497802734
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,3,0.1344213287035624
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,3,0.16293866435686746
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,3,0.14638933539390564
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,32,32767,4.286282539367676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,3,0.19221866130828857
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,3,0.13873599966367087
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,3,0.29013333717981976
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,64,32767,4.280762672424316
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,512,1,16,32767,4.297744115193685
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,3,0.13387200236320496
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,3,0.13608533143997192
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,7,0.14587199687957764
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,7,0.1366933286190033
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,7,0.19134400288263956
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,7,0.13460800051689148
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,7,0.16142933567365012
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,7,0.13295466701189676
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,7,0.28758400678634644
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,7,0.13729066650072733
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,7,0.13427199920018515
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,7,0.14658133188883463
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,7,0.16004266341527304
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,7,0.19004799922307333
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,7,0.1330453356107076
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,7,0.2874400019645691
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,15,0.13354667027791342
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,15,0.14436266819636026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,15,0.13146666685740152
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,15,0.15847466389338175
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,15,0.12946133812268576
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,15,0.1906506617863973
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,15,0.28433066606521606
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,15,0.1309386690457662
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,15,0.13385066390037537
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,15,0.14381866653760275
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,15,0.15874666968981424
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,15,0.13000532984733582
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,15,0.18885332345962524
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,15,0.28495999177296955
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,31,0.12563733259836832
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,31,0.13319466511408487
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,31,0.1399999956289927
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,31,0.12947733203570047
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,31,0.15637333194414774
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,31,0.278821329275767
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,31,0.18872533241907755
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,31,0.1329813301563263
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,31,0.13983466227849325
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,31,0.1291039983431498
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,31,0.1269866625467936
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,31,0.15634666879971823
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,31,0.18672533830006918
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,31,0.27853866418202716
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,63,0.13242133458455405
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,63,0.12839466333389282
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,63,0.15615466237068176
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,63,0.14265599846839905
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,63,0.2810666759808858
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,63,0.19312000274658203
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,63,0.1253653367360433
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,63,0.14130666851997375
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,63,0.1327946682771047
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,63,0.12893866499265036
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,63,0.15873600045839945
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,63,0.12718400359153748
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,63,0.19296000401178995
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,63,0.28217599789301556
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,127,0.14723199605941772
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,127,0.1599573294321696
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,127,0.15054933230082193
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,127,0.17715734243392944
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,127,0.3141760031382243
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,127,0.14495467146237692
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,127,0.21115734179814658
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,127,0.14422399799029031
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,127,0.14473066727320352
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,127,0.1601759990056356
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,127,0.15013333161671957
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,127,0.17730667193730673
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,127,0.3148639996846517
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,127,0.21400533119837442
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,255,0.18069332838058472
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,255,0.3695840040842692
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,255,0.18433600664138794
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,255,0.1942453384399414
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,255,0.2102880080540975
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,255,0.24605866273244223
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,255,0.17828800280888876
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,255,0.18159466981887817
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,255,0.1841599941253662
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,255,0.20843199888865152
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,255,0.1935840050379435
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,255,0.1746506690979004
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,255,0.36447465419769287
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,255,0.24613332748413086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,511,0.26108266909917194
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,511,0.27962666749954224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,511,0.24611733357111612
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,511,0.25410666068394977
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,511,0.24634132782618204
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,511,0.4737333456675212
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,511,0.3211519916852315
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,511,0.24902933835983276
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,511,0.2605866591135661
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,511,0.47412268320719403
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,511,0.2539253234863281
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,511,0.2460319995880127
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,511,0.2807413339614868
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,511,0.3184799949328105
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,1023,0.39742934703826904
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,1023,0.3873600165049235
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,1023,0.38253335158030194
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,1023,0.6800746917724609
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,1023,0.37945600350697833
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,1023,0.4146133263905843
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,1023,0.45792531967163086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,1023,0.6851200262705485
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,1023,0.38812800248463947
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,1023,0.3762293259302775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,1023,0.3823733329772949
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,1023,0.39851733048756915
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,1023,0.4163626829783122
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,1023,0.45821332931518555
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,2047,0.6545919974644979
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,2047,1.111967960993449
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,2047,0.6701386769612631
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,2047,0.6441919803619385
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,2047,0.6363093455632528
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,2047,0.7017333507537842
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,2047,0.7451252937316895
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,2047,1.1158506870269775
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,2047,0.6380426486333212
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,2047,0.6714080174763998
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,2047,0.6547360022862753
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,2047,0.6458719968795776
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,2047,0.7024532953898112
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,2047,0.7478559811909994
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,4095,1.310197353363037
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,4095,1.9710933367411296
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,4095,1.2109119892120361
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,4095,1.2520853678385417
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,4095,1.1672906875610352
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,4095,1.1871893405914307
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,4095,1.174341360727946
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,4095,1.9658080736796062
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,4095,1.3113760153452556
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,4095,1.252288023630778
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,4095,1.1889279683430989
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,4095,1.2106293042500813
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,4095,1.1733280022939045
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,4095,1.1646986802419026
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,8191,3.671519915262858
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,8191,2.398746649424235
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,8191,2.3195412953694663
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,8191,2.270106633504232
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,8191,2.2423200607299805
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,8191,2.2280160586039224
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,8191,2.215381304423014
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,8191,3.674650510152181
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,8191,2.3941493034362793
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,8191,2.274442672729492
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,8191,2.3194986979166665
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,8191,2.243669350941976
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,8191,2.2265920639038086
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,8191,2.218378702799479
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,1,16383,7.093285242716472
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,2,16383,4.517061233520508
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,8,16383,4.3793331782023115
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,4,16383,4.431338628133138
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,64,16383,4.31602128346761
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,32,16383,4.322938601175944
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,fp8,1024,1,16,16383,4.343471844991048
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,1,16383,7.088560104370117
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,2,16383,4.521365483601888
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,4,16383,4.431594530741374
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,8,16383,4.377557436625163
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,32,16383,4.325269381205241
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,64,16383,4.31492265065511
TRTLLM,0.20.0,NVIDIA H200,mla_generation,float16,float16,1024,1,16,16383,4.346735954284668
