framework,version,device,op_name,mla_dtype,kv_cache_dtype,batch_size,isl,tp_size,step,latency
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,2,0,0.013877333452304205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,16,0,0.012874666601419449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,8,0,0.012725333372751871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,1,0,0.014629332969586054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,4,0,0.013424000392357508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,64,0,0.012778667112191519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,1,0,0.01441066712141037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16,32,0,0.012383999923865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,2,0,0.01381333296497663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,32,0,0.01239466667175293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,16,0,0.01302933320403099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,4,0,0.013455999394257864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,8,0,0.012682666381200155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,1,0,0.014485333114862442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16,64,0,0.012400000045696894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,2,0,0.013914667069911957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,4,0,0.013616000612576803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,8,0,0.013306666165590286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,32,0,0.012378666549921036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,16,0,0.013306666165590286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,32,64,0,0.012261333564917246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,2,0,0.013973332941532135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,1,0,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,4,0,0.013631999492645264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,16,0,0.013568000247081121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,8,0,0.013888000200192133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,32,0,0.012928000340859095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,32,64,0,0.013264000415802002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,1,0,0.016506666938463848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,2,0,0.015290666371583939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,4,0,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,8,0,0.014858666807413101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,16,0,0.01441066712141037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,64,0,0.013877333452304205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,64,32,0,0.013946666071812311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,2,0,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,4,0,0.015226667126019796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,1,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,8,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,16,0,0.014490666488806406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,32,0,0.013951999445756277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,64,64,0,0.013888000200192133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,1,0,0.020053333292404812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,2,0,0.016650666793187458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,4,0,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,8,0,0.0161013330022494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,16,0,0.015770666301250458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,32,0,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,1,0,0.019738666713237762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,2,0,0.016949333250522614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,4,0,0.015834666788578033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,16,0,0.01584533353646596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,128,64,0,0.015253332753976187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,32,0,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,128,64,0,0.01524266724785169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,8,0,0.017802666872739792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,16,0,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,2,0,0.02015999952952067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,1,0,0.031173333525657654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,32,0,0.016602666427691776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,4,0,0.018506667266289394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,256,64,0,0.016303999970356624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,8,0,0.017674667139848072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,4,0,0.018506667266289394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,2,0,0.02025066688656807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,16,0,0.01710933322707812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,1,0,0.03153600047032038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,32,0,0.016650666793187458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,1,0,0.06910933554172516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,256,64,0,0.01666133354107539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,8,0,0.021583999196688335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,4,0,0.023152001202106476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,2,0,0.034304000437259674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,16,0,0.020853333175182343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,64,0,0.02014933278163274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,512,32,0,0.020928000410397846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,8,0,0.021536000072956085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,4,0,0.022917332748572033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,16,0,0.02164799968401591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,32,0,0.0206986665725708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,1,0,0.06865066786607106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,2,0,0.0340639998515447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,512,64,0,0.02011200040578842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,16,0,0.029205332199732464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,8,0,0.03070399910211563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,4,0,0.046309332052866616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,32,0,0.028517333169778187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,2,0,0.07788800199826558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,64,0,0.02794666588306427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1024,1,0,0.13105066617329916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,8,0,0.030666666726271313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,16,0,0.029077333708604176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,64,0,0.027834666272004444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,4,0,0.041936000188191734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,32,0,0.028757333755493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,2,0,0.07629866898059845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1024,1,0,0.13144532839457193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,16,0,0.03782399992148081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,32,0,0.036874666810035706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,8,0,0.0415040006240209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,4,0,0.06738133231798808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,64,0,0.0359946663180987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,2,0,0.14136000474294028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,1536,1,0,0.23069334030151367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,8,0,0.04154133299986521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,32,0,0.036714665591716766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,16,0,0.03738133360942205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,4,0,0.0678773323694865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,64,0,0.03626666714747747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,2,0,0.12985066572825113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,1536,1,0,0.23142399390538534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,8,0,0.057333335280418396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,16,0,0.04590400060017904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,4,0,0.10000000397364299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,32,0,0.04479999840259552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,64,0,0.04381866753101349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,2,0,0.19535466035207114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,2048,1,0,0.35922133922576904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,32,0,0.04453866680463155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,8,0,0.05668266614278158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,4,0,0.10128000378608704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,64,0,0.043562665581703186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,16,0,0.04614933331807455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,2,0,0.19785600900650024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,2048,1,0,0.3593226671218872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,16,0,0.06575466692447662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,8,0,0.10501866539319356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,64,0,0.059605335195859276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,4,0,0.20660267273585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,32,0,0.06067200005054474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,2,0,0.3700480063756307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,16,0,0.06602666775385539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,8,0,0.10559466481208801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,4,0,0.2092906634012858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,32,0,0.060975998640060425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,64,0,0.05955733358860016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,2,0,0.3702666759490967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,3072,1,0,0.7046666940053304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,16,0,0.08799466490745544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,32,0,0.07834666470686595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,64,0,0.07532800237337749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,3072,1,0,0.7074453035990397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,4,0,0.32874133189519245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,8,0,0.16858132680257162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,2,0,0.6133919954299927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,16,0,0.08826667070388794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,8,0,0.16051200032234192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,32,0,0.07921066880226135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,4,0,0.32921600341796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,64,0,0.07528533538182576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,2,0,0.6044426759084066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,4096,1,0,1.152618646621704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,16,0,0.1816426714261373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,8,0,0.3683840036392212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,64,0,0.10713066657384236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,32,0,0.11640000343322754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,4,0,0.6765279769897461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,4096,1,0,1.1613333225250244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,2,0,1.2530453205108643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,8,0,0.36395732561747235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,32,0,0.11506666739781697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,4,0,0.6485439936319987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,16,0,0.18488534291585287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,64,0,0.10776533683141072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,2,0,1.2502453327178955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,16,0,0.28250133991241455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,6144,1,0,2.422447999318441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,32,0,0.15190933148066202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,8,0,0.6105493307113647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,64,0,0.14390933513641357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,4,0,1.0895360310872395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,16,0,0.2784213423728943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,2,0,2.108026663462321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,6144,1,0,2.487173398335775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,8,0,0.5994666814804077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,4,0,1.124346653620402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,32,0,0.15076266725858053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,64,0,0.14338666200637817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,2,0,2.121514638264974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,16,0,0.5006080071131388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,8,0,0.875823974609375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,8192,1,0,4.272922515869141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,64,0,0.1800533334414164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,32,0,0.23825067281723022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,4,0,1.686405340830485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,8192,1,0,4.358394622802734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,16,0,0.5121813217798868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,8,0,0.8898666699727377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,2,0,3.269253412882487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,32,0,0.23918932676315308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,64,0,0.18258132537206015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,4,0,1.6522560119628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,16,0,0.6822826862335205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,2,0,3.3517494201660156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,8,0,1.2511040369669597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,32,0,0.3283626635869344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,64,0,0.21581333875656128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,10240,1,0,6.738869349161784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,4,0,2.3250880241394043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,16,0,0.6836427052815756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,8,0,1.247226635615031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,10240,1,0,6.563535690307617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,32,0,0.3291626572608948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,2,0,4.746693293253581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,64,0,0.21546133359273276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,4,0,2.3400746981302896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,16,0,1.1248106956481934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,2,0,4.55404790242513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,8,0,2.1241920789082847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,64,0,0.27819732824961346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,32,0,0.5248053471247355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,12288,1,0,9.559226353963217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,4,0,4.039877255757649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,16,0,1.1405759652455647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,8,0,2.0943466822306314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,32,0,0.5316853523254395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,2,0,8.514645258585611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,64,0,0.27989333868026733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,1,0,0.018453333526849747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,2,0,0.015306666493415833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,12288,1,0,9.709717432657877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,4,0,3.9913759231567383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,4,0,0.01440000037352244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,16,0,0.013733333597580591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,8,0,0.013728000223636627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,64,0,0.013221333424250284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16,32,0,0.012975999464591345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,1,0,0.018426666657129925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,2,0,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,4,0,0.01423466702302297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,8,0,0.013744000345468521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,16,0,0.013386666774749756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,32,0,0.012991999586423239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16,64,0,0.013301332791646322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,2,0,0.01504533365368843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,1,0,0.018816000471512478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,2,0,8.321157455444336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,4,0,0.0145066666106383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,16,0,0.013973332941532135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,8,0,0.014127999544143677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,32,0,0.013327999661366144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,32,64,0,0.013264000415802002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,1,0,0.019573333362738293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,4,0,0.01462399959564209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,2,0,0.01544533297419548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,8,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,16,0,0.013546666751305262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,32,0,0.01340266689658165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,32,64,0,0.013178666432698568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,1,0,0.021168000996112823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,4,0,0.015413332730531693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,2,0,0.016901332885026932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,8,0,0.014746667196353277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,16,0,0.014720000326633453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,32,0,0.014501333236694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,64,64,0,0.014202666779359182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,1,0,0.021226666867733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,2,0,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,4,0,0.015072000523408255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,8,0,0.014869333555301031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,16,0,0.014773332824309668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,32,0,0.014698666830857595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,64,64,0,0.01421333352724711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,1,16384,1,0,16.74795150756836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,2,0,0.020181333025296528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,1,0,0.036501333117485046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,4,0,0.01747200017174085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,8,0,0.016271999726692837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,16,0,0.01635733370979627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,32,0,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,128,64,0,0.015669333438078564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,4,0,0.017658667018016178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,2,0,0.019999999552965164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,1,0,0.032069332897663116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,8,0,0.01626666635274887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,16,0,0.016410666207472484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,32,0,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,128,64,0,0.015615999698638916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,4,0,0.0201706662774086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,8,0,0.0185759998857975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,2,0,0.03669333209594091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,16,0,0.017978666971127193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,1,0,0.06042666733264923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,32,0,0.017242666333913803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,256,64,0,0.01670933390657107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,16,0,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,8,0,0.018826667219400406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,2,0,0.032287999987602234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,1,0,0.061466669042905174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,32,0,0.017184000462293625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,64,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,256,4,0,0.021333334346612293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,8,0,0.023269332945346832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,16,0,0.02195200075705846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,32,0,0.021536000072956085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,4,0,0.03513599932193756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,64,0,0.02128000060717265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,2,0,0.06948266426722209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,1,16384,1,0,17.164384206136067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,512,1,0,0.10203733046849568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,16,0,0.022128000855445862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,8,0,0.023237332701683044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,4,0,0.03522666543722153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,32,0,0.021776000658671062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,2,0,0.06984533369541168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,64,0,0.021040000021457672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,512,1,0,0.10084799925486247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,32,0,0.030031998952229817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,16,0,0.03129599988460541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,64,0,0.02934933453798294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,8,0,0.0432640016078949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,4,0,0.07766399780909221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,2,0,0.15577600399653116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1024,1,0,0.23889599243799844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,32,0,0.030016000072161358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,16,0,0.031231999397277832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,8,0,0.04331733286380768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,4,0,0.07820266485214233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,64,0,0.028954667349656422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,2,0,0.13159466783205667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1024,1,0,0.23829332987467447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,16,0,0.043136000633239746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,8,0,0.06937600175539653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,64,0,0.03751466671625773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,32,0,0.03849066545565923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,4,0,0.13091199596722922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,2,0,0.23144533236821493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,16,0,0.045647998650868736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,8,0,0.06878933310508728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,32,0,0.03849600007136663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,4,0,0.13274133205413818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,64,0,0.03743999948104223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,1536,1,0,0.43480531374613446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,2,0,0.23170133431752524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,8,0,0.1043839951356252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,16,0,0.0582826683918635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,32,0,0.04826666911443075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,64,0,0.046037331223487854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,4,0,0.1978986660639445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,1536,1,0,0.43160001436869305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,2,0,0.3596426645914714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,32,0,0.04849066833655039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,16,0,0.058320000767707825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,8,0,0.10402133067448933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,4,0,0.19839467604955038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,64,0,0.045824001232783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,2,0,0.3578186829884847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,16,0,0.10745066404342651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,2048,1,0,0.731391986211141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,4,0,0.3746346632639567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,32,0,0.06898133456707001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,64,0,0.06384533147017162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,8,0,0.2126986583073934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,2048,1,0,0.7004640102386475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,2,0,0.6988693078358968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,8,0,0.21045867602030435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,16,0,0.10821333527565002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,32,0,0.06877866884072621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,64,0,0.06354666749636333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,4,0,0.3729546864827474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,16,0,0.1644426683584849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,3072,1,0,1.368842601776123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,2,0,0.706554651260376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,8,0,0.33081599076588947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,64,0,0.0831520011027654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,32,0,0.09206933776537578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,4,0,0.6152746677398682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,16,0,0.16581867138544717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,8,0,0.3325973351796468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,3072,1,0,1.4334719975789387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,32,0,0.09299199779828389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,4,0,0.607098658879598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,64,0,0.08349333206812541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,2,0,1.1730133692423503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,2,0,1.155509312947591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,16,0,0.3750400145848592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,32,0,0.1848319967587789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,8,0,0.6629439989725748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,64,0,0.12354666988054912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,4096,1,0,2.292133331298828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,4,0,1.248586654663086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,4096,1,0,2.2883307139078775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,8,0,0.6652266581853231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,16,0,0.3701866865158081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,32,0,0.18617600202560425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,64,0,0.12168000141779582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,4,0,1.2562506993611653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,2,0,2.475920041402181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,16,0,0.6134133338928223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,2,0,2.439530690511068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,8,0,1.1135199864705403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,32,0,0.29443732897440594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,64,0,0.15971733132998148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,4,0,2.122938632965088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,6144,1,0,5.047530810038249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,8,0,1.1096693674723308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,6144,1,0,5.025391896565755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,2,0,4.308970769246419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,4,0,2.121546745300293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,64,0,0.15961066881815592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,32,0,0.28833599885304767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,16,0,0.6220693190892538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,16,0,0.9142560164133707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,2,0,4.389573415120442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,8,0,1.6665013631184895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,64,0,0.24915200471878052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,32,0,0.5263893206914266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,8192,1,0,8.729546864827475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,4,0,3.245258649190267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,16,0,0.8855093320210775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,8,0,1.6580053965250652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,32,0,0.5077013174692789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,8192,1,0,8.920138676961264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,2,0,6.693648020426433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,64,0,0.2453920046488444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,4,0,3.3211841583251953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,16,0,1.2690880298614502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,8,0,2.3516906102498374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,2,0,6.71781857808431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,32,0,0.7156906922658285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,64,0,0.34013867378234863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,4,0,4.77511469523112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,10240,1,0,13.664267222086588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,8,0,2.362618605295817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,16,0,1.2485013008117676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,32,0,0.692138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,4,0,4.61735471089681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,64,0,0.34273600578308105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,2,0,9.605887730916342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,10240,1,0,13.52731196085612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,16,0,2.106762727101644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,2,0,9.61296526590983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,8,0,4.055941263834636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,64,0,0.5576906601587931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,32,0,1.1489866574605305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,4,0,8.564800262451172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,12288,1,0,19.07698694864909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,16,0,2.083669344584147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,8,0,4.039866765340169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,4,0,8.38813845316569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,64,0,0.5429173310597738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,32,0,1.1380266348520915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,2,0,16.951136271158855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,12288,1,0,19.773690541585285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,1,0,0.02518933266401291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,4,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,2,0,0.018874666343132656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,8,0,0.014378666877746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,16,0,0.013946666071812311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,32,0,0.013642666240533194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,16,64,0,0.012874666601419449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,1,0,0.025087999800841015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,2,0,0.01884799947341283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,4,0,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,16,0,0.01351999988158544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,8,0,0.014346666634082794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,64,0,0.013157332936922709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,16,32,0,0.013466666142145792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,1,0,0.02611733227968216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,2,0,0.018789333601792652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,8,0,0.014581333845853806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,4,0,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,16,0,0.014240000396966934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,32,0,0.013722666849692663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,32,64,0,0.013077333569526672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,1,0,0.026186667382717133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,2,0,0.018746666610240936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,4,0,0.015194666882356008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,8,0,0.014720000326633453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,16,0,0.013717333475748697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,32,0,0.013605333864688873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,32,64,0,0.013663999736309052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,1,0,0.03270933280388514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,8,0,0.015381333728631338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,2,0,0.021333334346612293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,4,0,0.017008000363906223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,16,0,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,32,0,0.014874666929244995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,64,64,0,0.014245333770910898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,4,0,0.017210666090250015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,2,0,0.02094399929046631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,1,0,0.03286933402220408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,8,0,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,16,0,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,32,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,64,64,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,2,0,17.0580317179362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,2,0,0.03252266595760981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,4,0,0.02027200038234393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,1,0,0.05561066667238871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,8,0,0.017898666361967724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,16,0,0.01621866722901662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,32,0,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,128,64,0,0.01589866727590561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,4,0,0.020527999848127365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,2,0,0.031727999448776245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,8,0,0.017397332936525345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,1,0,0.056517332792282104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,16,0,0.016789333273967106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,32,0,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,128,64,0,0.01570133368174235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,4,0,0.03324266771475474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,8,0,0.021002667645613354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,16,0,0.019029332945744198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,2,0,0.06047466893990835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,32,0,0.018277333428462345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,1,0,0.0956106682618459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,256,64,0,0.017114666601022083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,4,0,0.033002667129039764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,8,0,0.020714666694402695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,2,0,0.061050668358802795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,16,0,0.019023999571800232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,32,0,0.017717332889636356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,1,0,0.13507200280825296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,256,64,0,0.017301333447297413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,16,0,0.0240639994541804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,4,0,0.07138133545716603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,8,0,0.036506667733192444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,32,0,0.022757334013779957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,2,0,0.10247466961542766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,64,0,0.022170667846997578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,8,0,0.03688533355792364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,4,0,0.07127999762694041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,2,0,0.10147733489672343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,16,0,0.02439466615517934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,2,16384,1,0,33.969258626302086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,512,1,0,0.18398932615915933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,32,0,0.022917332748572033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,64,0,0.021802666286627453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,512,1,0,0.1857653260231018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,32,0,0.03364799916744232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,16,0,0.04535466432571411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,64,0,0.03139200061559677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,4,0,0.13482133547465006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,8,0,0.0811466674009959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,2,0,0.2849973241488139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,8,0,0.08035733302434285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,4,0,0.13520532846450806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,16,0,0.04516266783078512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,32,0,0.03403733422358831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,64,0,0.031317333380381264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1024,1,0,0.45229868094126385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,2,0,0.23992000023523966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,32,0,0.046629334489504494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,16,0,0.07191999753316243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,8,0,0.13358400265375772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,4,0,0.25916266441345215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1024,1,0,0.44939200083414715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,64,0,0.04077333211898804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,2,16384,1,0,33.847557067871094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,8,0,0.1329866647720337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,16,0,0.0717439999183019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,2,0,0.43751998742421466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,32,0,0.047637333472569786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,4,0,0.25945067405700684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,64,0,0.04102399945259094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,2,0,0.43406399091084796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,1536,1,0,0.8644266923268636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,16,0,0.10703466335932414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,8,0,0.21824532747268677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,32,0,0.06273066500822704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,4,0,0.3659253517786662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,64,0,0.053488001227378845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,1536,1,0,0.8442880312601725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,2,0,0.6953866481781006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,16,0,0.1095360020796458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,8,0,0.2030400037765503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,64,0,0.05394133428732554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,32,0,0.06272533535957336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,4,0,0.36346133550008136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,16,0,0.21741332610448202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,2,0,0.738912026087443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,2048,1,0,1.3572319348653157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,32,0,0.11534399787584941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,64,0,0.08001600205898285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,8,0,0.38023467858632404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,4,0,0.7141599655151367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,2048,1,0,1.3417332967122395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,16,0,0.21815999348958334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,8,0,0.38067734241485596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,2,0,1.3870399792989094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,32,0,0.11283733447392781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,64,0,0.07543466488520305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,4,0,0.7484052975972494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,16,0,0.3598560094833374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,8,0,0.6217973232269287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,2,0,1.3752106030782063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,32,0,0.1873813271522522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,64,0,0.10046933094660442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,4,0,1.1620906988779705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,3072,1,0,2.77837340037028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,16,0,0.34859200318654376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,8,0,0.6090133190155029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,2,0,2.3047733306884766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,32,0,0.1752693255742391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,4,0,1.1772960027058919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,64,0,0.09963200489679973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,3072,1,0,2.72377077738444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,16,0,0.6753546396891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,8,0,1.2501920064290364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,32,0,0.384058674176534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,2,0,2.3815733591715493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,64,0,0.1950826644897461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,4096,1,0,4.748048146565755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,4,0,2.4433600107828775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,16,0,0.6779893239339193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,4096,1,0,4.809546788533528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,8,0,1.2601546446482341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,32,0,0.38791998227437335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,64,0,0.2015786568323771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,4,0,2.4419733683268228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,2,0,5.115802764892578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,16,0,1.140389362970988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,8,0,2.1069812774658203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,32,0,0.624837319056193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,2,0,5.254538536071777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,64,0,0.30109333992004395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,4,0,4.195834795633952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,6144,1,0,10.577679951985678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,8,0,2.1276960372924805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,4,0,4.115536053975423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,2,0,8.709269205729166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,32,0,0.6185226837793986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,16,0,1.1316800117492676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,1,0,0.03791466603676478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,6144,1,0,10.36959457397461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,2,0,0.025061334172884624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,64,0,0.3022560079892476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,4,0,0.01825599993268649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,16,0,0.014511999984582266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,8,0,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,32,0,0.014021333307027817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,16,64,0,0.01340266689658165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,1,0,0.03863999992609024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,4,0,0.01814933369557063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,2,0,0.025248001019159954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,8,0,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,16,0,0.014127999544143677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,32,0,0.013663999736309052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,16,64,0,0.01381333296497663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,1,0,0.04181333382924398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,2,0,0.02533866713444392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,4,0,0.01821333294113477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,8,0,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,32,0,0.014058666924635569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,32,64,0,0.013845333208640417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,2,0,0.025279998779296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,1,0,0.04178666571776072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,4,0,0.018687999496857326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,8,0,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,16,0,0.014325333138306936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,32,0,0.013978666315476099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,32,64,0,0.013845333208640417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,2,0,8.951552073160807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,1,0,0.0547626664241155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,4,0,0.02139200021823247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,8,0,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,2,0,0.03342933456103007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,16,0,0.015263999501864115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,32,0,0.015365333606799444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,64,64,0,0.014858666807413101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,4,0,0.021482666333516438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,2,0,0.0329120010137558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,8,0,0.01704000060757001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,1,0,0.054511999090512596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,32,0,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,16,0,0.015439999600251516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,64,64,0,0.014917333920796713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,4,0,0.03325333446264267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,2,0,0.05612266560395559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,8,0,0.02081599955757459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,16,0,0.017642666896184284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,32,0,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,1,0,0.09970133503278096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,128,64,0,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,16,0,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,4,0,0.0391839991013209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,8,0,0.02060266708334287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,2,0,0.0561653325955073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,1,0,0.09957333405812581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,32,0,0.016949333250522614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,128,64,0,0.016645333419243496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,4,0,0.06352533400058746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,16,0,0.021488000949223835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,32,0,0.019962667177120846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,8,0,0.03436266630887985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,2,0,0.09658132990201314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,64,0,0.019152000546455383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,256,1,0,0.2566559910774231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,8,0,0.03425599883000056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,4,0,0.0620000014702479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,16,0,0.021477334201335907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,2,0,0.09635733564694722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,32,0,0.01970133309563001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,64,0,0.01860800012946129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,256,1,0,0.17153600851694742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,16,0,0.039290666580200195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,8,0,0.10195733110109965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,4,0,0.10285333792368571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,32,0,0.026202666262785595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,4,8192,1,0,17.826075236002605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,64,0,0.024362665911515553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,2,0,0.18665067354838052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,32,0,0.02584533393383026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,16,0,0.039034667114416756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,8,0,0.07543999950091045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,4,0,0.10372266173362732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,512,1,0,0.3440693219502767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,64,0,0.02405333270629247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,2,0,0.18533867597579956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,32,0,0.049925332268079124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,16,0,0.08577066659927368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,8,0,0.1362826625506083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,512,1,0,0.4992693265279134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,64,0,0.03882666677236557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,4,0,0.24343466758728027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,4,8192,1,0,18.08293279012044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,16,0,0.08553066849708557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,8,0,0.13824533422787985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,2,0,0.4559839963912964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,32,0,0.05022933085759481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,64,0,0.03926933308442434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,4,0,0.2892319957415263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,2,0,0.45125865936279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,16,0,0.15666666626930237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,8,0,0.2408693234125773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,32,0,0.07809066772460938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,64,0,0.05990933378537496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1024,1,0,0.8897493680318197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,4,0,0.44895466168721515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,16,0,0.14247999588648477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1024,1,0,0.8928746382395426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,8,0,0.24077866474787393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,2,0,0.8497280279795328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,32,0,0.07869866490364075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,64,0,0.05297600229581197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,4,0,0.48553065458933514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,16,0,0.22923733790715536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,8,0,0.37304532527923584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,2,0,0.8426132996877035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,32,0,0.14113600055376688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,64,0,0.07106666763623555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,4,0,0.7127573490142822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,1536,1,0,1.685226599375407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,8,0,0.37694935003916424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,16,0,0.21041067441304526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,2,0,1.35915740331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,1536,1,0,1.673514684041341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,64,0,0.07081066568692525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,32,0,0.11764267086982727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,4,0,0.7017172972361246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,16,0,0.3938613335291545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,8,0,0.7365067005157471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,2,0,1.4498027165730794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,32,0,0.2299626668294271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,64,0,0.1264746685822805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,2048,1,0,2.7117013931274414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,4,0,1.3969333966573079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,16,0,0.3930240074793498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,8,0,0.7243359883626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,2048,1,0,2.697632153828939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,32,0,0.22924800713857016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,64,0,0.13132799665133157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,2,0,2.7243305842081704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,4,0,1.41921599706014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,16,0,0.6437866687774658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,2,0,2.733776092529297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,8,0,1.1871893405914307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,32,0,0.3586239814758301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,64,0,0.18872533241907755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,4,0,2.3089653650919595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,3072,1,0,5.64243761698405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,8,0,1.1923147042592366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,16,0,0.6313973267873129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,2,0,4.577978769938151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,4,0,2.32203737894694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,32,0,0.35620800654093426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,64,0,0.18874132633209229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,1,0,0.06188266475995382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,4,0,0.02436800052722295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,2,0,0.037434667348861694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,3072,1,0,5.895402908325195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,8,0,0.01811733345190684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,16,0,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,32,0,0.014181333283583323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,16,64,0,0.01394133393963178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,4,0,0.023930666347344715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,2,0,0.03606933355331421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,1,0,0.07003733515739441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,16,0,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,32,0,0.014053333550691605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,16,64,0,0.013712000101804733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,2,0,0.04173333446184794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,1,0,0.0813973347345988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,4,0,0.02532266577084859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,8,0,0.018543999642133713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,16,0,0.01562133307258288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,32,0,0.014501333236694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,32,64,0,0.014474666366974512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,1,0,0.06947200000286102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,2,0,0.04136000076929728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,8,0,0.019066666563351948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,2,0,4.819984118143718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,4,0,0.02515200028816859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,16,0,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,32,0,0.014522666732470194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,32,64,0,0.01414399966597557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,16,0,0.017632000148296356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,32,0,0.015775999675194424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,2,0,0.059119999408721924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,8,0,0.021333334346612293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,4,0,0.033386667569478355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,1,0,0.09622400005658467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,64,64,0,0.015546667079130808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,16,0,0.017818666994571686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,8,0,0.021488000949223835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,4,0,0.0395359992980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,2,0,0.05471999943256378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,32,0,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,1,0,0.09699199597040813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,64,64,0,0.01571200042963028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,16,0,0.021397332350413006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,4,0,0.05737066765626272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,8,0,0.03473066786924998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,2,0,0.1377066671848297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,32,0,0.018826667219400406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,64,0,0.018016000588734944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,128,1,0,0.18726933002471924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,8,0,0.034661332766215004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,4,0,0.07620800038178761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,16,0,0.021503999829292297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,32,0,0.018453333526849747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,64,0,0.018042666216691334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,2,0,0.10009066263834636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,128,1,0,0.18398400147755942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,32,0,0.02309333284695943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,4,0,0.09875200192133586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,8,0,0.06502399841944377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,16,0,0.03789333254098892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,8,4096,1,0,9.668981552124023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,2,0,0.17273600896199545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,64,0,0.02107733239730199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,16,0,0.0376800000667572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,8,0,0.06587733328342438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,32,0,0.023898666103680927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,4,0,0.0983679989973704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,256,1,0,0.4965226650238037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,64,0,0.021007999777793884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,2,0,0.17251733938852945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,32,0,0.044010668992996216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,16,0,0.07921599845091502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,8,0,0.1486293375492096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,256,1,0,0.3240213394165039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,64,0,0.03214933226505915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,4,0,0.18914133310317993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,16,0,0.07958933214346568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,2,0,0.35068265597025555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,8,0,0.1076746682325999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,8,4096,1,0,9.641498565673828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,4,0,0.18795200188954672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,32,0,0.04496533175309499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,64,0,0.03183466692765554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,2,0,0.5032639900843302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,16,0,0.14577066898345947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,8,0,0.2523840069770813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,512,1,0,0.6693173249562582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,32,0,0.1276533305644989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,64,0,0.058970664938290916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,4,0,0.46245865027109784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,512,1,0,0.6733386516571045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,8,0,0.2529226740201314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,16,0,0.14524799585342407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,32,0,0.09321600198745728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,64,0,0.05834133426348368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,4,0,0.5520106554031372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,2,0,0.8955466747283936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,16,0,0.25221333901087445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,2,0,0.887834628423055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,8,0,0.5010453462600708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,32,0,0.15227199594179788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,64,0,0.0904319981733958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,4,0,0.875653346379598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1024,1,0,1.7826666831970215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,16,0,0.25150400400161743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,8,0,0.4569653272628784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,2,0,1.694101333618164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,64,0,0.08945600191752116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,4,0,0.8565759658813477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,32,0,0.152837336063385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1024,1,0,1.7848000526428223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,16,0,0.38714667161305744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,2,0,1.6957279841105144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,8,0,0.7159679730733236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,32,0,0.2251733342806498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,64,0,0.13045333822568259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,4,0,1.366762638092041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,1536,1,0,3.3547948201497397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,8,0,0.7112906773885092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,16,0,0.3858400185902913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,4,0,1.3744959831237793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,32,0,0.22404799858729044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,64,0,0.130431999762853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,2,0,2.7795947392781577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,2,0,0.06145066519578298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,4,0,0.03619199991226196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,1,0,0.1109226644039154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,8,0,0.02421333392461141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,1536,1,0,3.7320534388224282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,32,0,0.015077333897352219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,16,0,0.017871999492247898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,16,64,0,0.014175999909639359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,4,0,0.03589866558710734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,2,0,0.0709440012772878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,1,0,0.1095199982325236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,8,0,0.024005333582560223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,16,0,0.018112000077962875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,32,0,0.015024000157912573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,16,64,0,0.01451733335852623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,8,0,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,2,0,0.06985066831111908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,16,0,0.019061333189407986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,1,0,0.1336799959341685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,4,0,0.04165333261092504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,32,0,0.015626666446526844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,32,64,0,0.014752000570297241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,2,0,0.06966400146484375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,8,0,0.025653332471847534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,1,0,0.15237866838773093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,4,0,0.04133866727352142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,16,0,0.019093333433071773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,32,0,0.015791999797026317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,2,0,2.738623936971029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,32,64,0,0.014831999937693277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,4,0,0.05643733342488607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,16,0,0.022495999932289124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,32,0,0.018272000054518383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,8,0,0.035418666899204254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,2,0,0.09842666983604431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,64,0,0.016293333222468693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,64,1,0,0.2584906617800395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,8,0,0.03524799893299738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,4,0,0.05607999861240387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,16,0,0.022202665607134502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,2,0,0.09865066409111023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,32,0,0.018565333137909572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,64,0,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,64,1,0,0.1805866758028666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,8,0,0.0802346666653951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,16,0,0.03696533292531967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,4,0,0.10619200269381206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,32,0,0.02325333406527837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,64,0,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,2,0,0.18480000893274942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,4,0,0.10302399595578511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,16,0,0.0367999995748202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,8,0,0.07997333506743114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,128,1,0,0.348410685857137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,2,0,0.1906986633936564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,64,0,0.019760000209013622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,32,0,0.023210667073726654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,16,2048,1,0,5.505365371704102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,16,0,0.07007466753323872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,128,1,0,0.349509318669637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,32,0,0.04455466568470001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,8,0,0.10230933626492818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,64,0,0.029296000798543293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,4,0,0.1766666571299235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,16,0,0.07081600030263265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,2,0,0.325546662012736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,8,0,0.10196800033251445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,4,0,0.17708265781402588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,32,0,0.04373333354791006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,64,0,0.02945599953333537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,2,0,0.32573866844177246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,16,2048,1,0,5.4352264404296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,256,1,0,0.9721866448720297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,8,0,0.19720532496770224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,16,0,0.11658666531244914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,32,0,0.09071999788284302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,64,0,0.07063466807206471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,4,0,0.3535360097885132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,256,1,0,0.6279306809107462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,32,0,0.08961600065231323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,16,0,0.11540266871452332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,8,0,0.2783679962158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,64,0,0.054042667150497437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,4,0,0.3562133312225342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,2,0,0.6742186546325684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,16,0,0.2649120092391968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,2,0,0.6813173294067383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,8,0,0.4806133508682251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,32,0,0.1601599951585134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,64,0,0.10955733060836792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,512,1,0,1.3521973292032878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,4,0,0.9096373716990153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,16,0,0.26609599590301514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,512,1,0,1.3655734062194824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,8,0,0.47915732860565186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,64,0,0.10957866907119751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,32,0,0.16010666886965433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,1,0,0.2487786610921224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,4,0,0.9177227020263672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,2,0,0.1146613359451294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,4,0,0.06224533418814341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,8,0,0.03626666714747747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,16,0,0.02455466737349828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,32,0,0.01833600054184596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,16,64,0,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,2,0,2.1848959922790527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,8,0,0.035962666074434914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,4,0,0.06259733438491821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,2,0,0.11105066537857056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,16,0,0.02492266645034154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,1,0,0.25171732902526855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,32,0,0.0182239996890227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,16,64,0,0.015200000256299973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,4,0,0.07101866602897644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,16,0,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,8,0,0.04286933441956838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,2,0,0.153738667567571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,32,0,0.01958400011062622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,64,0,0.016458666572968166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,32,1,0,0.26626133918762207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,16,0,0.02754133443037669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,8,0,0.04311466713746389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,2,0,0.1525813341140747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,4,0,0.0701333334048589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,32,0,0.019866666446129482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,1,0,0.2645333409309387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,32,64,0,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,2,0,1.780618667602539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,16,0,0.03758399933576584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,8,0,0.05880000193913778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,32,0,0.02480533222357432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,4,0,0.10167466600735982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,2,0,0.18316266934076944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,64,0,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,16,0,0.03812800099452337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,8,0,0.05875733494758606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,4,0,0.10207999746004741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,64,1,0,0.5004160006841024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,32,0,0.02421333392461141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,64,0,0.019904000063737232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,2,0,0.18405866622924805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,16,0,0.06576533118883769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,64,1,0,0.345909317334493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,8,0,0.10672533512115479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,64,0,0.028688001135985058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,32,0,0.0421013335386912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,4,0,0.19142399231592813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,2,0,0.5049226681391398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,32,1024,1,0,3.5534559885660806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,8,0,0.10683199763298035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,16,0,0.08718400200208028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,32,0,0.04243200023969015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,4,0,0.19139732917149863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,64,0,0.029103999336560566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,2,0,0.3550453186035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,128,1,0,0.6772212982177734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,32,0,0.07985599835713704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,16,0,0.11010133226712544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,8,0,0.18363734086354574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,128,1,0,0.680506706237793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,64,0,0.05292266607284546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,4,0,0.33371198177337646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,16,0,0.11028800408045451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,32,1024,1,0,3.5875574747721353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,32,0,0.07920533418655396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,2,0,0.6315733194351196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,8,0,0.1833546757698059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,4,0,0.33314667145411175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,64,0,0.0540533314148585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,16,0,0.2132213314374288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,2,0,0.6325173377990723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,8,0,0.37116265296936035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,32,0,0.13183466593424478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,64,0,0.10468799869219463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,256,1,0,2.033199946085612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,256,1,0,1.2733226617177327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,4,0,0.6940159797668457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,8,0,0.5381866693496704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,32,0,0.13119999567667642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,64,0,0.15919466813405356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,16,0,0.22005333503087363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,4,0,0.6980640093485514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,4,0,0.11149332920710246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,2,0,0.2576799988746643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,8,0,0.06407466530799866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,2,0,1.361237366994222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,1,0,0.5050026575724283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,16,0,0.036981334288915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,32,0,0.025434667865435284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,16,64,0,0.019215999792019527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,16,0,0.036661334335803986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,8,0,0.06945600112279256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,4,0,0.11160533626874287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,2,0,0.25057599941889447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,32,0,0.025018667181332905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,64,0,0.01899733394384384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,16,1,0,0.5129013458887736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,8,0,0.087226668993632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,16,0,0.04644800225893656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,4,0,0.1350986659526825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,2,0,0.26544533173243207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,32,0,0.029711998999118805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,64,0,0.021365332106749218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,2,0,1.37063995997111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,8,0,0.07334400216738383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,32,1,0,0.5322293440500895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,4,0,0.15706666310628256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,16,0,0.04773866633574168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,32,0,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,64,0,0.02125866711139679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,2,0,0.26637333631515503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,32,1,0,0.5295146703720093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,16,0,0.06388799846172333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,8,0,0.10652800401051839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,4,0,0.1916373372077942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,64,0,0.02938133229811986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,32,0,0.04241600135962168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,2,0,0.3523893356323242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,8,0,0.10498666763305664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,4,0,0.19342400630315146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,32,0,0.04264533519744873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,16,0,0.06447466711203258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,64,0,0.02938133229811986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,2,0,0.34902934233347577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,64,512,1,0,2.6776746114095054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,64,1,0,0.9817226727803549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,32,0,0.10016000270843506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,16,0,0.11877866586049397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,8,0,0.20272000630696616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,64,1,0,0.676256020863851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,64,0,0.0682826687892278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,4,0,0.3665173451105754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,64,512,1,0,2.7131627400716147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,8,0,0.2792746623357137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,16,0,0.1172160009543101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,32,0,0.07610133290290833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,2,0,0.6912639935811361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,64,0,0.06823466718196869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,4,0,0.3667999903361003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,16,0,0.2000160018603007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,2,0,0.7009920279184977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,8,0,0.535370667775472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,32,0,0.12545599540074667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,64,0,0.09619200229644775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,128,1,0,1.3637866973876953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,4,0,0.6483253240585327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,128,1,0,1.364426612854004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,16,0,0.20004266500473022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,8,0,0.3505386511484782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,32,0,0.13809067010879517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,64,0,0.09430399537086487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,4,0,0.6486560106277466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,4,0,0.2584639986356099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,2,0,0.5127893288930258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,8,0,0.11540800333023071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,2,0,1.2896959781646729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,16,0,0.06692799925804138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,32,0,0.04055466751257578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,64,0,0.02737066646416982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,16,1,0,1.0135626792907715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,16,0,0.06859200199445088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,8,0,0.11539733409881592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,4,0,0.2555306752522786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,64,0,0.027280000348885853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,32,0,0.03881066789229711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,2,0,0.5117013454437256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,16,1,0,1.0137653350830078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,8,0,0.13933866222699484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,2,0,1.2888212998708088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,4,0,0.26838932434717816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,16,0,0.07864533364772797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,32,0,0.05089066425959269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,64,0,0.035360001027584076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,2,0,0.5527679920196533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,8,0,0.13937600453694662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,16,0,0.0783733328183492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,4,0,0.2754613359769185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,32,0,0.051130667328834534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,64,0,0.034117333590984344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,2,0,0.5379946629206339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,32,1,0,1.0617706775665283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,16,0,0.11528000235557556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,8,0,0.1967680056889852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,32,1,0,1.069386641184489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,32,0,0.07319466769695282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,4,0,0.37204798062642414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,64,0,0.05206400156021118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,8,0,0.19829867283503214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,2,0,0.6878613630930582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,128,256,1,0,2.544538656870524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,4,0,0.37463998794555664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,16,0,0.11400000254313152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,32,0,0.07364266614119212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,64,0,0.05287466446558634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,128,256,1,0,4.071109453837077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,2,0,0.6889920234680176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,16,0,0.31631465752919513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,32,0,0.13173333803812662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,64,1,0,1.3535839716593425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,8,0,0.37841065724690753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,64,0,0.12232533097267151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,4,0,0.7201653321584066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,64,1,0,1.3487839698791504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,8,0,0.5384426514307658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,16,0,0.21535466114679971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,32,0,0.14192000031471252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,64,0,0.12180800239245097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,4,0,0.7220693429311117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,2,0,1.3962079683939617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,2,0,1.3942933082580566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,fp8,256,128,1,0,2.7487147649129233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,mla_context,float16,float16,256,128,1,0,2.721088091532389
